From 269cda016dd6ea92b66e55ebe283965924e67bc1 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:55 +0000 Subject: [PATCH 0001/1172] lang/c/msgpack: added Messagepack, a binary-based efficient data interchange format. git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@48 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- README | 20 + c/Makefile | 17 + c/bench.c | 338 +++++++ c/bench_inline.c | 329 +++++++ c/msgpack | 1 + c/pack.c | 40 + c/pack.h | 38 + c/pack_inline.h | 14 + c/unpack.c | 56 ++ c/unpack.h | 59 ++ c/unpack_context.h | 30 + c/unpack_inline.c | 82 ++ cpp/unpack.cc | 30 + cpp/unpack.h | 126 +++ cpp/unpack_context.h | 13 + msgpack/pack/inline_context.h | 2 + msgpack/pack/inline_impl.h | 287 ++++++ msgpack/unpack/callback.h | 24 + msgpack/unpack/inline_context.h | 52 + msgpack/unpack/inline_impl.h | 438 +++++++++ ruby/Makefile | 153 +++ ruby/bench.rb | 60 ++ ruby/extconf.rb | 4 + ruby/gem.sh | 19 + ruby/gem/AUTHORS | 1 + ruby/gem/History.txt | 0 ruby/gem/License.txt | 13 + ruby/gem/Manifest.txt | 26 + ruby/gem/PostInstall.txt | 1 + ruby/gem/Rakefile | 4 + ruby/gem/config/hoe.rb | 75 ++ ruby/gem/config/requirements.rb | 15 + ruby/gem/lib/msgpack/version.rb | 9 + ruby/gem/script/console | 10 + ruby/gem/script/destroy | 14 + ruby/gem/script/generate | 14 + ruby/gem/script/txt2html | 82 ++ ruby/gem/setup.rb | 1585 +++++++++++++++++++++++++++++++ ruby/gem/tasks/deployment.rake | 34 + ruby/gem/tasks/environment.rake | 7 + ruby/pack.c | 131 +++ ruby/pack.h | 26 + ruby/pack_inline.h | 33 + ruby/rbinit.c | 29 + ruby/test_format.rb | 128 +++ ruby/test_pack.rb | 56 ++ ruby/unpack.c | 202 ++++ ruby/unpack.h | 26 + ruby/unpack_context.h | 35 + ruby/unpack_inline.c | 81 ++ 50 files changed, 4869 insertions(+) create mode 100644 README create mode 100644 c/Makefile create mode 100644 c/bench.c create mode 100644 c/bench_inline.c create mode 120000 c/msgpack create mode 100644 c/pack.c create mode 100644 c/pack.h create mode 100644 c/pack_inline.h create mode 100644 c/unpack.c create mode 100644 c/unpack.h create mode 100644 c/unpack_context.h create mode 100644 c/unpack_inline.c create mode 100644 cpp/unpack.cc create mode 100644 cpp/unpack.h create mode 100644 cpp/unpack_context.h create mode 100644 msgpack/pack/inline_context.h create mode 100644 msgpack/pack/inline_impl.h create mode 100644 msgpack/unpack/callback.h create mode 100644 msgpack/unpack/inline_context.h create mode 100644 msgpack/unpack/inline_impl.h create mode 100644 ruby/Makefile create mode 100644 ruby/bench.rb create mode 100644 ruby/extconf.rb create mode 100755 ruby/gem.sh create mode 100644 ruby/gem/AUTHORS create mode 100644 ruby/gem/History.txt create mode 100644 ruby/gem/License.txt create mode 100644 ruby/gem/Manifest.txt create mode 100644 ruby/gem/PostInstall.txt create mode 100644 ruby/gem/Rakefile create mode 100644 ruby/gem/config/hoe.rb create mode 100644 ruby/gem/config/requirements.rb create mode 100644 ruby/gem/lib/msgpack/version.rb create mode 100755 ruby/gem/script/console create mode 100755 ruby/gem/script/destroy create mode 100755 ruby/gem/script/generate create mode 100755 ruby/gem/script/txt2html create mode 100644 ruby/gem/setup.rb create mode 100644 ruby/gem/tasks/deployment.rake create mode 100644 ruby/gem/tasks/environment.rake create mode 100644 ruby/pack.c create mode 100644 ruby/pack.h create mode 100644 ruby/pack_inline.h create mode 100644 ruby/rbinit.c create mode 100644 ruby/test_format.rb create mode 100644 ruby/test_pack.rb create mode 100644 ruby/unpack.c create mode 100644 ruby/unpack.h create mode 100644 ruby/unpack_context.h create mode 100644 ruby/unpack_inline.c diff --git a/README b/README new file mode 100644 index 0000000..31a482a --- /dev/null +++ b/README @@ -0,0 +1,20 @@ +MessagePack +----------- +Binary-based efficient data interchange format. + + + +Copyright (C) 2008 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/c/Makefile b/c/Makefile new file mode 100644 index 0000000..d4b2226 --- /dev/null +++ b/c/Makefile @@ -0,0 +1,17 @@ + +CFLAGS = -I.. -Wall -g -O4 +LDFLAGS = -L. + +all: bench + +bench: pack.o unpack.o unpack_inline.o bench.o pack.h pack_inline.h unpack.h unpack_context.h + $(CC) $(LDFLAGS) unpack.o unpack_inline.o pack.o bench.o -lyajl_s -o $@ + +bench_inline: pack.o bench_inline.o pack.h pack_inline.h + $(CC) $(LDFLAGS) pack.o bench_inline.o -lyajl_s -o $@ + +.PHONY: clean +clean: + $(RM) unpack.o unpack_inline.o pack.o test.o bench.o bench_inline.o + $(RM) bench bench_inline + diff --git a/c/bench.c b/c/bench.c new file mode 100644 index 0000000..f27350a --- /dev/null +++ b/c/bench.c @@ -0,0 +1,338 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + + +static struct timeval g_timer; + +void reset_timer() +{ + gettimeofday(&g_timer, NULL); +} + +double show_timer() +{ + struct timeval endtime; + gettimeofday(&endtime, NULL); + double sec = (endtime.tv_sec - g_timer.tv_sec) + + (double)(endtime.tv_usec - g_timer.tv_usec) / 1000 / 1000; + printf("%f sec\n", sec); + return sec; +} + + +static int reformat_null(void * ctx) { return 1; } +static int reformat_boolean(void * ctx, int boolean) { return 1; } +static int reformat_number(void * ctx, const char * s, unsigned int l) { return 1; } +static int reformat_string(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } +static int reformat_map_key(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } +static int reformat_start_map(void * ctx) { return 1; } +static int reformat_end_map(void * ctx) { return 1; } +static int reformat_start_array(void * ctx) { return 1; } +static int reformat_end_array(void * ctx) { return 1; } + + +static void* unpack_unsigned_int_8(void* data, uint8_t d) { return NULL; } +static void* unpack_unsigned_int_16(void* data, uint16_t d) { return NULL; } +static void* unpack_unsigned_int_32(void* data, uint32_t d) { return NULL; } +static void* unpack_unsigned_int_64(void* data, uint64_t d) { return NULL; } +static void* unpack_signed_int_8(void* data, int8_t d) { return NULL; } +static void* unpack_signed_int_16(void* data, int16_t d) { return NULL; } +static void* unpack_signed_int_32(void* data, int32_t d) { return NULL; } +static void* unpack_signed_int_64(void* data, int64_t d) { return NULL; } +static void* unpack_float(void* data, float d) { return NULL; } +static void* unpack_double(void* data, double d) { return NULL; } +static void* unpack_nil(void* data) { return NULL; } +static void* unpack_true(void* data) { return NULL; } +static void* unpack_false(void* data) { return NULL; } +static void* unpack_array_start(void* data, unsigned int n) { return NULL; } +static void unpack_array_item(void* data, void* c, void* o) { } +static void* unpack_map_start(void* data, unsigned int n) { return NULL; } +static void unpack_map_item(void* data, void* c, void* k, void* v) { } +static void* unpack_string(void* data, const void* b, size_t l) { return NULL; } +static void* unpack_raw(void* data, const void* b, size_t l) { /*printf("unpack raw %p %lu\n",b,l);*/ return NULL; } + +typedef struct { + size_t allocated; + size_t length; + char* buffer; +} pack_buffer; + +static const size_t PACK_INITIAL_BUFFER_SIZE = 512; + +static void pack_buffer_init(pack_buffer* data) +{ + data->buffer = malloc(PACK_INITIAL_BUFFER_SIZE); + data->length = 0; + data->allocated = PACK_INITIAL_BUFFER_SIZE; +} + +static void pack_buffer_reset(pack_buffer* data) +{ + data->buffer = realloc(data->buffer, PACK_INITIAL_BUFFER_SIZE); + data->allocated = PACK_INITIAL_BUFFER_SIZE; + data->length = 0; +} + +static void pack_buffer_free(pack_buffer* data) +{ + free(data->buffer); +} + +static void pack_append_buffer(void* user, const unsigned char* b, unsigned int l) +{ + pack_buffer* data = (pack_buffer*)user; + if(data->allocated - data->length < l) { + data->buffer = realloc(data->buffer, data->allocated*2); + data->allocated *= 2; + } + memcpy(data->buffer + data->length, b, l); + data->length += l; +} + + +static const unsigned int TASK_INT_NUM = 1<<24; +static const unsigned int TASK_STR_LEN = 1<<15; +//static const unsigned int TASK_INT_NUM = 1<<20; +//static const unsigned int TASK_STR_LEN = 1<<12; +static const char* TASK_STR_PTR; + + +void bench_json(void) +{ + puts("== JSON =="); + + + yajl_gen_config gcfg = {0, NULL}; + yajl_gen g = yajl_gen_alloc(&gcfg); + + yajl_parser_config hcfg = { 0, 0 }; + yajl_callbacks callbacks = { + reformat_null, + reformat_boolean, + NULL, + NULL, + reformat_number, + reformat_string, + reformat_start_map, + reformat_map_key, + reformat_end_map, + reformat_start_array, + reformat_end_array + }; + yajl_handle h = yajl_alloc(&callbacks, &hcfg, NULL); + + + double sec; + const unsigned char * buf; + unsigned int len; + + + puts("generate integer"); + reset_timer(); + { + unsigned int i; + yajl_gen_array_open(g); + for(i=0; i < TASK_INT_NUM; ++i) { + yajl_gen_integer(g, i); + } + yajl_gen_array_close(g); + } + sec = show_timer(); + + yajl_gen_get_buf(g, &buf, &len); + printf("%u KB\n", len / 1024); + printf("%f MB/s\n", len / sec / 1024 / 1024); + + puts("----"); + puts("parse integer"); + reset_timer(); + { + yajl_status stat = yajl_parse(h, buf, len); + if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { + unsigned char * str = yajl_get_error(h, 1, buf, len); + fprintf(stderr, (const char *) str); + } + } + sec = show_timer(); + + printf("%f MB/s\n", len / sec / 1024 / 1024); + + + //yajl_gen_clear(g); + yajl_gen_free(g); + g = yajl_gen_alloc(&gcfg); + yajl_free(h); + h = yajl_alloc(&callbacks, &hcfg, NULL); + + + puts("----"); + puts("generate string"); + reset_timer(); + { + unsigned int i; + yajl_gen_array_open(g); + for(i=0; i < TASK_STR_LEN; ++i) { + yajl_gen_string(g, (const unsigned char*)TASK_STR_PTR, i); + } + yajl_gen_array_close(g); + } + sec = show_timer(); + + yajl_gen_get_buf(g, &buf, &len); + printf("%u KB\n", len / 1024); + printf("%f MB/s\n", len / sec / 1024 / 1024); + + puts("----"); + puts("parse string"); + reset_timer(); + { + yajl_status stat = yajl_parse(h, buf, len); + if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { + unsigned char * str = yajl_get_error(h, 1, buf, len); + fprintf(stderr, (const char *) str); + } + } + sec = show_timer(); + + printf("%f MB/s\n", len / sec / 1024 / 1024); + + + yajl_gen_free(g); + yajl_free(h); +} + +void bench_msgpack(void) +{ + puts("== MessagePack =="); + + + pack_buffer mpkbuf; + pack_buffer_init(&mpkbuf); + msgpack_pack_t* mpk = msgpack_pack_new( + &mpkbuf, pack_append_buffer); + + msgpack_unpack_callback cb = { + unpack_unsigned_int_8, + unpack_unsigned_int_16, + unpack_unsigned_int_32, + unpack_unsigned_int_64, + unpack_signed_int_8, + unpack_signed_int_16, + unpack_signed_int_32, + unpack_signed_int_64, + unpack_float, + unpack_double, + unpack_nil, + unpack_true, + unpack_false, + unpack_array_start, + unpack_array_item, + unpack_map_start, + unpack_map_item, + unpack_string, + unpack_raw, + }; + msgpack_unpack_t* mupk = msgpack_unpack_new(NULL, &cb); + + + double sec; + size_t len; + const char* buf; + + + puts("pack integer"); + reset_timer(); + { + unsigned int i; + msgpack_pack_array(mpk, TASK_INT_NUM); + for(i=0; i < TASK_INT_NUM; ++i) { + msgpack_pack_unsigned_int(mpk, i); + } + } + sec = show_timer(); + + len = mpkbuf.length; + buf = mpkbuf.buffer; + printf("%lu KB\n", len / 1024); + printf("%f MB/s\n", len / sec / 1024 / 1024); + + puts("----"); + puts("unpack integer"); + reset_timer(); + { + size_t off = 0; + int ret = msgpack_unpack_execute(mupk, buf, len, &off); + if(ret < 0) { + fprintf(stderr, "Parse error.\n"); + } else if(ret == 0) { + fprintf(stderr, "Not finished.\n"); + } + } + sec = show_timer(); + + printf("%f MB/s\n", len / sec / 1024 / 1024); + + + pack_buffer_reset(&mpkbuf); + msgpack_unpack_reset(mupk); + + + puts("----"); + puts("pack string"); + reset_timer(); + { + unsigned int i; + msgpack_pack_array(mpk, TASK_STR_LEN); + for(i=0; i < TASK_STR_LEN; ++i) { + msgpack_pack_raw(mpk, TASK_STR_PTR, i); + } + } + sec = show_timer(); + + len = mpkbuf.length; + buf = mpkbuf.buffer; + printf("%lu KB\n", len / 1024); + printf("%f MB/s\n", len / sec / 1024 / 1024); + + puts("----"); + puts("unpack string"); + reset_timer(); + { + size_t off = 0; + int ret = msgpack_unpack_execute(mupk, buf, len, &off); + if(ret < 0) { + fprintf(stderr, "Parse error.\n"); + } else if(ret == 0) { + fprintf(stderr, "Not finished.\n"); + } + } + sec = show_timer(); + + printf("%f MB/s\n", len / sec / 1024 / 1024); + + + msgpack_unpack_free(mupk); + msgpack_pack_free(mpk); + pack_buffer_free(&mpkbuf); +} + +int main(int argc, char* argv[]) +{ + char* str = malloc(TASK_STR_LEN); + memset(str, 'a', TASK_STR_LEN); + TASK_STR_PTR = str; + + bench_msgpack(); + bench_json(); + + return 0; +} + + diff --git a/c/bench_inline.c b/c/bench_inline.c new file mode 100644 index 0000000..2901508 --- /dev/null +++ b/c/bench_inline.c @@ -0,0 +1,329 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + + +static struct timeval g_timer; + +void reset_timer() +{ + gettimeofday(&g_timer, NULL); +} + +double show_timer() +{ + struct timeval endtime; + gettimeofday(&endtime, NULL); + double sec = (endtime.tv_sec - g_timer.tv_sec) + + (double)(endtime.tv_usec - g_timer.tv_usec) / 1000 / 1000; + printf("%f sec\n", sec); + return sec; +} + + +static int reformat_null(void * ctx) { return 1; } +static int reformat_boolean(void * ctx, int boolean) { return 1; } +static int reformat_number(void * ctx, const char * s, unsigned int l) { return 1; } +static int reformat_string(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } +static int reformat_map_key(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } +static int reformat_start_map(void * ctx) { return 1; } +static int reformat_end_map(void * ctx) { return 1; } +static int reformat_start_array(void * ctx) { return 1; } +static int reformat_end_array(void * ctx) { return 1; } + + +typedef void* msgpack_object; + +typedef struct { +} msgpack_unpack_context; + +#include "msgpack/unpack/inline_context.h" + +static inline void* msgpack_unpack_init(msgpack_unpack_context* x) { return NULL; } +static inline void* msgpack_unpack_unsigned_int_8(msgpack_unpack_context* x, uint8_t d) { return NULL; } +static inline void* msgpack_unpack_unsigned_int_16(msgpack_unpack_context* x, uint16_t d) { return NULL; } +static inline void* msgpack_unpack_unsigned_int_32(msgpack_unpack_context* x, uint32_t d) { return NULL; } +static inline void* msgpack_unpack_unsigned_int_64(msgpack_unpack_context* x, uint64_t d) { return NULL; } +static inline void* msgpack_unpack_signed_int_8(msgpack_unpack_context* x, int8_t d) { return NULL; } +static inline void* msgpack_unpack_signed_int_16(msgpack_unpack_context* x, int16_t d) { return NULL; } +static inline void* msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int32_t d) { return NULL; } +static inline void* msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d) { return NULL; } +static inline void* msgpack_unpack_float(msgpack_unpack_context* x, float d) { return NULL; } +static inline void* msgpack_unpack_double(msgpack_unpack_context* x, double d) { return NULL; } +static inline void* msgpack_unpack_big_int(msgpack_unpack_context* x, const void* b, unsigned int l) { return NULL; } +static inline void* msgpack_unpack_big_float(msgpack_unpack_context* x, const void* b, unsigned int l) { return NULL; } +static inline void* msgpack_unpack_nil(msgpack_unpack_context* x) { return NULL; } +static inline void* msgpack_unpack_true(msgpack_unpack_context* x) { return NULL; } +static inline void* msgpack_unpack_false(msgpack_unpack_context* x) { return NULL; } +static inline void* msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned int n) { return NULL; } +static inline void msgpack_unpack_array_item(msgpack_unpack_context* x, void* c, void* o) { } +static inline void* msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n) { return NULL; } +static inline void msgpack_unpack_map_item(msgpack_unpack_context* x, void* c, void* k, void* v) { } +static inline void* msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l) { return NULL; } +static inline void* msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, size_t l) { return NULL; } + +#include "msgpack/unpack/inline_impl.h" + +typedef struct { + size_t allocated; + size_t length; + char* buffer; +} pack_buffer; + +static const size_t PACK_INITIAL_BUFFER_SIZE = 512; + +static void pack_buffer_init(pack_buffer* data) +{ + data->buffer = malloc(PACK_INITIAL_BUFFER_SIZE); + data->length = 0; + data->allocated = PACK_INITIAL_BUFFER_SIZE; +} + +static void pack_buffer_reset(pack_buffer* data) +{ + data->buffer = realloc(data->buffer, PACK_INITIAL_BUFFER_SIZE); + data->allocated = PACK_INITIAL_BUFFER_SIZE; + data->length = 0; +} + +static void pack_buffer_free(pack_buffer* data) +{ + free(data->buffer); +} + +static void pack_append_buffer(void* user, const unsigned char* b, unsigned int l) +{ + pack_buffer* data = (pack_buffer*)user; + if(data->allocated - data->length < l) { + data->buffer = realloc(data->buffer, data->allocated*2); + data->allocated *= 2; + } + memcpy(data->buffer + data->length, b, l); + data->length += l; +} + + +static const unsigned int TASK_INT_NUM = 1<<24; +static const unsigned int TASK_STR_LEN = 1<<15; +//static const unsigned int TASK_INT_NUM = 1<<20; +//static const unsigned int TASK_STR_LEN = 1<<12; +static const char* TASK_STR_PTR; + + +void bench_json(void) +{ + puts("== JSON =="); + + + yajl_gen_config gcfg = {0, NULL}; + yajl_gen g = yajl_gen_alloc(&gcfg); + + yajl_parser_config hcfg = { 0, 0 }; + yajl_callbacks callbacks = { + reformat_null, + reformat_boolean, + NULL, + NULL, + reformat_number, + reformat_string, + reformat_start_map, + reformat_map_key, + reformat_end_map, + reformat_start_array, + reformat_end_array + }; + yajl_handle h = yajl_alloc(&callbacks, &hcfg, NULL); + + + double sec; + const unsigned char * buf; + unsigned int len; + + + puts("generate integer"); + reset_timer(); + { + unsigned int i; + yajl_gen_array_open(g); + for(i=0; i < TASK_INT_NUM; ++i) { + yajl_gen_integer(g, i); + } + yajl_gen_array_close(g); + } + sec = show_timer(); + + yajl_gen_get_buf(g, &buf, &len); + printf("%u KB\n", len / 1024); + printf("%f Mbps\n", len / sec / 1024 / 1024); + + puts("----"); + puts("parse integer"); + reset_timer(); + { + yajl_status stat = yajl_parse(h, buf, len); + if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { + unsigned char * str = yajl_get_error(h, 1, buf, len); + fprintf(stderr, (const char *) str); + } + } + sec = show_timer(); + + printf("%f Mbps\n", len / sec / 1024 / 1024); + + + //yajl_gen_clear(g); + yajl_gen_free(g); + g = yajl_gen_alloc(&gcfg); + yajl_free(h); + h = yajl_alloc(&callbacks, &hcfg, NULL); + + + puts("----"); + puts("generate string"); + reset_timer(); + { + unsigned int i; + yajl_gen_array_open(g); + for(i=0; i < TASK_STR_LEN; ++i) { + yajl_gen_string(g, (const unsigned char*)TASK_STR_PTR, i); + } + yajl_gen_array_close(g); + } + sec = show_timer(); + + yajl_gen_get_buf(g, &buf, &len); + printf("%u KB\n", len / 1024); + printf("%f Mbps\n", len / sec / 1024 / 1024); + + puts("----"); + puts("parse string"); + reset_timer(); + { + yajl_status stat = yajl_parse(h, buf, len); + if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { + unsigned char * str = yajl_get_error(h, 1, buf, len); + fprintf(stderr, (const char *) str); + } + } + sec = show_timer(); + + printf("%f Mbps\n", len / sec / 1024 / 1024); + + + yajl_gen_free(g); + yajl_free(h); +} + +void bench_msgpack(void) +{ + puts("== MessagePack =="); + + + pack_buffer mpkbuf; + pack_buffer_init(&mpkbuf); + msgpack_pack_t* mpk = msgpack_pack_new( + &mpkbuf, pack_append_buffer); + + msgpack_unpacker mupk; + msgpack_unpacker_init(&mupk); + + double sec; + size_t len; + const char* buf; + + + puts("pack integer"); + reset_timer(); + { + unsigned int i; + msgpack_pack_array(mpk, TASK_INT_NUM); + for(i=0; i < TASK_INT_NUM; ++i) { + msgpack_pack_unsigned_int(mpk, i); + } + } + sec = show_timer(); + + len = mpkbuf.length; + buf = mpkbuf.buffer; + printf("%lu KB\n", len / 1024); + printf("%f Mbps\n", len / sec / 1024 / 1024); + + puts("----"); + puts("unpack integer"); + reset_timer(); + { + size_t off = 0; + int ret = msgpack_unpacker_execute(&mupk, buf, len, &off); + if(ret < 0) { + fprintf(stderr, "Parse error.\n"); + } else if(ret == 0) { + fprintf(stderr, "Not finished.\n"); + } + } + sec = show_timer(); + + printf("%f Mbps\n", len / sec / 1024 / 1024); + + + pack_buffer_reset(&mpkbuf); + msgpack_unpacker_init(&mupk); + + + puts("----"); + puts("pack string"); + reset_timer(); + { + unsigned int i; + msgpack_pack_array(mpk, TASK_STR_LEN); + for(i=0; i < TASK_STR_LEN; ++i) { + msgpack_pack_raw(mpk, TASK_STR_PTR, i); + } + } + sec = show_timer(); + + len = mpkbuf.length; + buf = mpkbuf.buffer; + printf("%lu KB\n", len / 1024); + printf("%f Mbps\n", len / sec / 1024 / 1024); + + puts("----"); + puts("unpack string"); + reset_timer(); + { + size_t off = 0; + int ret = msgpack_unpacker_execute(&mupk, buf, len, &off); + if(ret < 0) { + fprintf(stderr, "Parse error.\n"); + } else if(ret == 0) { + fprintf(stderr, "Not finished.\n"); + } + } + sec = show_timer(); + + printf("%f Mbps\n", len / sec / 1024 / 1024); + + + msgpack_pack_free(mpk); + pack_buffer_free(&mpkbuf); +} + +int main(int argc, char* argv[]) +{ + char* str = malloc(TASK_STR_LEN); + memset(str, 'a', TASK_STR_LEN); + TASK_STR_PTR = str; + + bench_msgpack(); + //bench_json(); + + return 0; +} + + + diff --git a/c/msgpack b/c/msgpack new file mode 120000 index 0000000..945c9b4 --- /dev/null +++ b/c/msgpack @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/c/pack.c b/c/pack.c new file mode 100644 index 0000000..05bd38d --- /dev/null +++ b/c/pack.c @@ -0,0 +1,40 @@ +/* + * MessagePack packing routine for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pack_inline.h" +#include +#include + +msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_callback_t callback) +{ + msgpack_pack_t* ctx = calloc(1, sizeof(msgpack_pack_t)); + if(!ctx) { return NULL; } + ctx->data = data; + ctx->callback = callback; + return ctx; +} + +void msgpack_pack_free(msgpack_pack_t* ctx) +{ + free(ctx); +} + +static inline void msgpack_pack_append_buffer(msgpack_pack_t* ctx, const unsigned char* b, unsigned int l) +{ + ctx->callback(ctx->data, b, l); +} + diff --git a/c/pack.h b/c/pack.h new file mode 100644 index 0000000..8107d9f --- /dev/null +++ b/c/pack.h @@ -0,0 +1,38 @@ +#ifndef MSGPACK_PACK_H__ +#define MSGPACK_PACK_H__ + +#include +#include + +typedef void (*msgpack_pack_callback_t)(void* data, const unsigned char* b, unsigned int i); + +typedef struct { + void* data; + msgpack_pack_callback_t callback; +} msgpack_pack_t; + +msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_callback_t callback); +void msgpack_pack_free(msgpack_pack_t* ctx); + +void msgpack_pack_int(msgpack_pack_t* ctx, int d); +void msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); +void msgpack_pack_unsigned_int_8(msgpack_pack_t* ctx, uint8_t d); +void msgpack_pack_unsigned_int_16(msgpack_pack_t* ctx, uint16_t d); +void msgpack_pack_unsigned_int_32(msgpack_pack_t* ctx, uint32_t d); +void msgpack_pack_unsigned_int_64(msgpack_pack_t* ctx, uint64_t d); +void msgpack_pack_signed_int_8(msgpack_pack_t* ctx, int8_t d); +void msgpack_pack_signed_int_16(msgpack_pack_t* ctx, int16_t d); +void msgpack_pack_signed_int_32(msgpack_pack_t* ctx, int32_t d); +void msgpack_pack_signed_int_64(msgpack_pack_t* ctx, int64_t d); +void msgpack_pack_float(msgpack_pack_t* ctx, float d); +void msgpack_pack_double(msgpack_pack_t* ctx, double d); +void msgpack_pack_nil(msgpack_pack_t* ctx); +void msgpack_pack_true(msgpack_pack_t* ctx); +void msgpack_pack_false(msgpack_pack_t* ctx); +void msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); +void msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); +void msgpack_pack_string(msgpack_pack_t* ctx, const char* b); +void msgpack_pack_raw(msgpack_pack_t* ctx, const void* b, size_t l); + +#endif /* msgpack/pack.h */ + diff --git a/c/pack_inline.h b/c/pack_inline.h new file mode 100644 index 0000000..d943464 --- /dev/null +++ b/c/pack_inline.h @@ -0,0 +1,14 @@ +#ifndef PACK_INLINE_H__ +#define PACK_INLINE_H__ + +#include "pack.h" + +typedef msgpack_pack_t* msgpack_pack_context; + +static inline void msgpack_pack_append_buffer(msgpack_pack_t* x, const unsigned char* b, unsigned int l); + +#include "msgpack/pack/inline_impl.h" + + +#endif /* pack_inline.h */ + diff --git a/c/unpack.c b/c/unpack.c new file mode 100644 index 0000000..9fe3695 --- /dev/null +++ b/c/unpack.c @@ -0,0 +1,56 @@ +/* + * MessagePack packing routine for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "unpack.h" +#include "unpack_context.h" +#include + +msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback) +{ + msgpack_unpacker* ctx; + ctx = (msgpack_unpacker*)calloc(1, sizeof(msgpack_unpacker)); + if(ctx == NULL) { return NULL; } + msgpack_unpacker_init(ctx); + ((msgpack_unpack_t*)ctx)->data = data; + ((msgpack_unpack_t*)ctx)->callback = *callback; + return (msgpack_unpack_t*)ctx; +} + +void msgpack_unpack_free(msgpack_unpack_t* ctx) +{ + free((msgpack_unpacker*)ctx); +} + +void* msgpack_unpack_data(msgpack_unpack_t* ctx) +{ + return msgpack_unpacker_data((msgpack_unpacker*)ctx); +} + +void msgpack_unpack_reset(msgpack_unpack_t* ctx) +{ + msgpack_unpack_t x = ((msgpack_unpacker*)ctx)->user; + msgpack_unpacker_init((msgpack_unpacker*)ctx); + ((msgpack_unpacker*)ctx)->user = x; +} + +int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off) +{ + return msgpack_unpacker_execute( + (msgpack_unpacker*)ctx, + data, len, off); +} + diff --git a/c/unpack.h b/c/unpack.h new file mode 100644 index 0000000..3230a0d --- /dev/null +++ b/c/unpack.h @@ -0,0 +1,59 @@ +/* + * MessagePack unpacking routine for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_UNPACK_H__ +#define MSGPACK_UNPACK_H__ + +#include +#include + +typedef struct { + void* (*unpack_unsigned_int_8)(void* data, uint8_t d); + void* (*unpack_unsigned_int_16)(void* data, uint16_t d); + void* (*unpack_unsigned_int_32)(void* data, uint32_t d); + void* (*unpack_unsigned_int_64)(void* data, uint64_t d); + void* (*unpack_signed_int_8)(void* data, int8_t d); + void* (*unpack_signed_int_16)(void* data, int16_t d); + void* (*unpack_signed_int_32)(void* data, int32_t d); + void* (*unpack_signed_int_64)(void* data, int64_t d); + void* (*unpack_float)(void* data, float d); + void* (*unpack_double)(void* data, double d); + void* (*unpack_nil)(void* data); + void* (*unpack_true)(void* data); + void* (*unpack_false)(void* data); + void* (*unpack_array_start)(void* data, unsigned int n); + void (*unpack_array_item)(void* data, void* c, void* o); + void* (*unpack_map_start)(void* data, unsigned int n); + void (*unpack_map_item)(void* data, void* c, void* k, void* v); + void* (*unpack_string)(void* data, const void* b, size_t l); + void* (*unpack_raw)(void* data, const void* b, size_t l); +} msgpack_unpack_callback; + +typedef struct { + void* data; + msgpack_unpack_callback callback; +} msgpack_unpack_t; + +msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback); +void msgpack_unpack_free(msgpack_unpack_t* ctx); +void msgpack_unpack_reset(msgpack_unpack_t* ctx); + +int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off); +void* msgpack_unpack_data(msgpack_unpack_t* ctx); + +#endif /* msgpack/unpack.h */ + diff --git a/c/unpack_context.h b/c/unpack_context.h new file mode 100644 index 0000000..7337c9e --- /dev/null +++ b/c/unpack_context.h @@ -0,0 +1,30 @@ +/* + * MessagePack unpacking routine for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef UNPACK_CONTEXT_H__ +#define UNPACK_CONTEXT_H__ + +#include "unpack.h" + +typedef void* msgpack_object; + +typedef msgpack_unpack_t msgpack_unpack_context; + +#include "msgpack/unpack/inline_context.h" + +#endif /* unpack_context.h */ + diff --git a/c/unpack_inline.c b/c/unpack_inline.c new file mode 100644 index 0000000..3525468 --- /dev/null +++ b/c/unpack_inline.c @@ -0,0 +1,82 @@ +/* + * MessagePack unpacking routine for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "unpack_context.h" + +static inline void* msgpack_unpack_init(msgpack_unpack_t* x) +{ return NULL; } + +static inline void* msgpack_unpack_unsigned_int_8(msgpack_unpack_t* x, uint8_t d) +{ return x->callback.unpack_unsigned_int_8(x->data, d); } + +static inline void* msgpack_unpack_unsigned_int_16(msgpack_unpack_t* x, uint16_t d) +{ return x->callback.unpack_unsigned_int_16(x->data, d); } + +static inline void* msgpack_unpack_unsigned_int_32(msgpack_unpack_t* x, uint32_t d) +{ return x->callback.unpack_unsigned_int_32(x->data, d); } + +static inline void* msgpack_unpack_unsigned_int_64(msgpack_unpack_t* x, uint64_t d) +{ return x->callback.unpack_unsigned_int_64(x->data, d); } + +static inline void* msgpack_unpack_signed_int_8(msgpack_unpack_t* x, int8_t d) +{ return x->callback.unpack_signed_int_8(x->data, d); } + +static inline void* msgpack_unpack_signed_int_16(msgpack_unpack_t* x, int16_t d) +{ return x->callback.unpack_signed_int_16(x->data, d); } + +static inline void* msgpack_unpack_signed_int_32(msgpack_unpack_t* x, int32_t d) +{ return x->callback.unpack_signed_int_32(x->data, d); } + +static inline void* msgpack_unpack_signed_int_64(msgpack_unpack_t* x, int64_t d) +{ return x->callback.unpack_signed_int_64(x->data, d); } + +static inline void* msgpack_unpack_float(msgpack_unpack_t* x, float d) +{ return x->callback.unpack_float(x->data, d); } + +static inline void* msgpack_unpack_double(msgpack_unpack_t* x, double d) +{ return x->callback.unpack_double(x->data, d); } + +static inline void* msgpack_unpack_nil(msgpack_unpack_t* x) +{ return x->callback.unpack_nil(x->data); } + +static inline void* msgpack_unpack_true(msgpack_unpack_t* x) +{ return x->callback.unpack_true(x->data); } + +static inline void* msgpack_unpack_false(msgpack_unpack_t* x) +{ return x->callback.unpack_false(x->data); } + +static inline void* msgpack_unpack_array_start(msgpack_unpack_t* x, unsigned int n) +{ return x->callback.unpack_array_start(x->data, n); } + +static inline void msgpack_unpack_array_item(msgpack_unpack_t* x, void* c, void* o) +{ x->callback.unpack_array_item(x->data, c, o); } + +static inline void* msgpack_unpack_map_start(msgpack_unpack_t* x, unsigned int n) +{ return x->callback.unpack_map_start(x->data, n); } + +static inline void msgpack_unpack_map_item(msgpack_unpack_t* x, void* c, void* k, void* v) +{ x->callback.unpack_map_item(x->data, c, k, v); } + +static inline void* msgpack_unpack_string(msgpack_unpack_t* x, const void* b, size_t l) +{ return x->callback.unpack_string(x->data, b, l); } + +static inline void* msgpack_unpack_raw(msgpack_unpack_t* x, const void* b, size_t l) +{ return x->callback.unpack_raw(x->data, b, l); } + + +#include "msgpack/unpack/inline_impl.h" + diff --git a/cpp/unpack.cc b/cpp/unpack.cc new file mode 100644 index 0000000..ce57e67 --- /dev/null +++ b/cpp/unpack.cc @@ -0,0 +1,30 @@ +#include "unpack.h" +#include "unpack_context.h" +#include + +msgpack_unpack_t* msgpack_unpack_new(void) +{ + msgpack_unpacker* ctx; + ctx = (msgpack_unpacker*)calloc(1, sizeof(msgpack_unpacker)); + if(ctx == NULL) { return NULL; } + msgpack_unpacker_init(ctx); + return (msgpack_unpack_t*)ctx; +} + +void msgpack_unpack_free(msgpack_unpack_t* ctx) +{ + free((msgpack_unpacker*)ctx); +} + +int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off) +{ + return msgpack_unpacker_execute( + (msgpack_unpacker*)ctx, + data, len, off); +} + +void* msgpack_unpack_data(msgpack_unpack_t* ctx) +{ + return msgpack_unpacker_data((msgpack_unpacker*)ctx); +} + diff --git a/cpp/unpack.h b/cpp/unpack.h new file mode 100644 index 0000000..cf7a168 --- /dev/null +++ b/cpp/unpack.h @@ -0,0 +1,126 @@ +#ifndef MSGPACK_UNPACK_H__ +#define MSGPACK_UNPACK_H__ + +#include +#include + +namespace MessagePack { + +class Unpacker { + class object { + template + object(const T& x) : m_obj(new holder(x)) {} + }; + + class type_error : public std::exception { }; + class cast_error : public type_error { }; + class overflow_error : public type_error { }; + class underflow_error : public type_error { }; + + struct object { + virtual ~object() {} + virtual bool isnil() const { return false; } + virtual bool xbool() const { throw cast_error(); } + virtual uint8_t xu8() const { throw cast_error(); } + virtual uint16_t xu16() const { throw cast_error(); } + virtual uint32_t xu32() const { throw cast_error(); } + virtual uint64_t xu64() const { throw cast_error(); } + virtual int8_t xi8() const { throw cast_error(); } + virtual int16_t xi16() const { throw cast_error(); } + virtual int32_t xi32() const { throw cast_error(); } + virtual int64_t xi64() const { throw cast_error(); } + virtual float xfloat() const { throw cast_error(); } + virtual double xdouble() const { throw cast_error(); } + virtual std::map& xmap() const { throw cast_error(); } + virtual std::string& xstring() const { throw cast_error(); } + virtual std::pair xraw() const { throw cast_error(); } + public: + template + inline void check_overflow(X x) { + if(std::numeric_limits::max() < x) { throw overflow_error(); } + } + template + inline void check_underflow(X x) { + if(std::numeric_limits::min() > x) { throw overflow_error(); } + } + }; + +private: + struct object_nil : object { + bool isnil() const { return true; } + }; + + struct object_true : object { + bool xbool() const { return true; } + }; + + struct object_false : object { + bool xbool() const { return false; } + }; + + struct object_u8 : object { + object_u8(uint8_t val) : m_val(val) {} + uint8_t xu8() const { return m_val; } + uint16_t xu16() const { return static_cast(m_val); } + uint32_t xu32() const { return static_cast(m_val); } + uint64_t xu64() const { return static_cast(m_val); } + int8_t xi8() const { check_overflow(m_val); return m_val; } + int16_t xi16() const { return static_cast(m_val); } + int32_t xi32() const { return static_cast(m_val); } + int64_t xi64() const { return static_cast(m_val); } + private: + uint8_t m_val; + }; + + struct object_u16 : object { + object_u16(uint16_t val) : m_val(val) {} + uint8_t xu8() const { check_overflow(m_val); return m_val; } + uint16_t xu16() const { return m_val; } + uint32_t xu32() const { return static_cast(m_val); } + uint64_t xu64() const { return static_cast(m_val); } + int8_t xi8() const { check_overflow< int8_t>(m_val); return m_val; } + int16_t xi16() const { check_overflow(m_val); return m_val; } + int32_t xi32() const { return static_cast(m_val); } + int64_t xi64() const { return static_cast(m_val); } + private: + uint16_t m_val; + }; + + ... +}; + +} // namespace MessagePack + +typedef struct { + void* (*unpack_unsigned_int_8)(void* data, uint8_t d); + void* (*unpack_unsigned_int_16)(void* data, uint16_t d); + void* (*unpack_unsigned_int_32)(void* data, uint32_t d); + void* (*unpack_unsigned_int_64)(void* data, uint64_t d); + void* (*unpack_signed_int_8)(void* data, int8_t d); + void* (*unpack_signed_int_16)(void* data, int16_t d); + void* (*unpack_signed_int_32)(void* data, int32_t d); + void* (*unpack_signed_int_64)(void* data, int64_t d); + void* (*unpack_float)(void* data, float d); + void* (*unpack_double)(void* data, double d); + void* (*unpack_big_int)(void* data, const void* b, unsigned int l); + void* (*unpack_big_float)(void* data, const void* b, unsigned int l); + void* (*unpack_nil)(void* data); + void* (*unpack_true)(void* data); + void* (*unpack_false)(void* data); + void* (*unpack_array_start)(void* data, unsigned int n); + void (*unpack_array_item)(void* data, void* c, void* o); + void* (*unpack_map_start)(void* data, unsigned int n); + void (*unpack_map_item)(void* data, void* c, void* k, void* v); + void* (*unpack_string)(void* data, const void* b, size_t l); + void* (*unpack_raw)(void* data, const void* b, size_t l); + void* data; +} msgpack_unpack_t; + +msgpack_unpack_t* msgpack_unpack_new(void); +void msgpack_unpack_free(msgpack_unpack_t* ctx); +int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off); +void* msgpack_unpack_data(msgpack_unpack_t* ctx); + +#endif /* msgpack/unpack.h */ + + diff --git a/cpp/unpack_context.h b/cpp/unpack_context.h new file mode 100644 index 0000000..caf2271 --- /dev/null +++ b/cpp/unpack_context.h @@ -0,0 +1,13 @@ +#ifndef UNPACK_CONTEXT_H__ +#define UNPACK_CONTEXT_H__ + +#include "unpack.h" + +typedef void* msgpack_object; + +typedef msgpack_unpack_t msgpack_unpack_context; + +#include "msgpack/unpack/inline_context.h" + +#endif /* unpack_context.h */ + diff --git a/msgpack/pack/inline_context.h b/msgpack/pack/inline_context.h new file mode 100644 index 0000000..139597f --- /dev/null +++ b/msgpack/pack/inline_context.h @@ -0,0 +1,2 @@ + + diff --git a/msgpack/pack/inline_impl.h b/msgpack/pack/inline_impl.h new file mode 100644 index 0000000..5c4bfed --- /dev/null +++ b/msgpack/pack/inline_impl.h @@ -0,0 +1,287 @@ +/* + * MessagePack packing routine + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_INLINE_IMPL_H__ +#define MSGPACK_PACK_INLINE_IMPL_H__ + +#include +#include + +#ifdef __LITTLE_ENDIAN__ + +#define STORE_16(d) \ + ((char*)&d)[1], ((char*)&d)[0] + +#define STORE_32(d) \ + ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] + +#define STORE_64(d) \ + ((char*)&d)[7], ((char*)&d)[6], ((char*)&d)[5], ((char*)&d)[4], \ + ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] + +#elif __BIG_ENDIAN__ + +#define STORE_16(d) \ + ((char*)&d)[2], ((char*)&d)[3] + +#define STORE_32(d) \ + ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3] + +#define STORE_32(d) \ + ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3], \ + ((char*)&d)[4], ((char*)&d)[5], ((char*)&d)[6], ((char*)&d)[7] + +#endif + + +/* + * Integer + */ + +// wrapper +inline void msgpack_pack_int(msgpack_pack_context x, int d) +{ + if(d < -32) { + if(d < -32768) { // signed 32 + const unsigned char buf[5] = {0xd2, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } else if(d < -128) { // signed 16 + const unsigned char buf[3] = {0xd1, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { // signed 8 + const unsigned char buf[2] = {0xd0, (uint8_t)d}; + msgpack_pack_append_buffer(x, buf, 2); + } + } else if(d < 128) { // fixnum + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); + } else { + if(d < 256) { + // unsigned 8 + const unsigned char buf[2] = {0xcc, (uint8_t)d}; + msgpack_pack_append_buffer(x, buf, 2); + } else if(d < 65536) { + // unsigned 16 + const unsigned char buf[3] = {0xcd, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + // unsigned 32 + const unsigned char buf[5] = {0xce, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } + } +} + +// wrapper +inline void msgpack_pack_unsigned_int(msgpack_pack_context x, unsigned int d) +{ + if(d < 128) { + // fixnum + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); + } else if(d < 256) { + // unsigned 8 + const unsigned char buf[2] = {0xcc, (uint8_t)d}; + msgpack_pack_append_buffer(x, buf, 2); + } else if(d < 65536) { + // unsigned 16 + const unsigned char buf[3] = {0xcd, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + // unsigned 32 + const unsigned char buf[5] = {0xce, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + +inline void msgpack_pack_unsigned_int_8(msgpack_pack_context x, uint8_t d) +{ + if(d < 128) { + msgpack_pack_append_buffer(x, &d, 1); + } else { + const unsigned char buf[2] = {0xcc, d}; + msgpack_pack_append_buffer(x, buf, 2); + } +} + +inline void msgpack_pack_unsigned_int_16(msgpack_pack_context x, uint16_t d) +{ + const unsigned char buf[3] = {0xcd, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +inline void msgpack_pack_unsigned_int_32(msgpack_pack_context x, uint32_t d) +{ + const unsigned char buf[5] = {0xce, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +inline void msgpack_pack_unsigned_int_64(msgpack_pack_context x, uint64_t d) +{ + // FIXME + const unsigned char buf[9] = {0xcf, STORE_64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +inline void msgpack_pack_signed_int_8(msgpack_pack_context x, int8_t d) +{ + if(d > 0) { + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); + } else if(d >= -32) { + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); + } else { + const unsigned char buf[2] = {0xd0, d}; + msgpack_pack_append_buffer(x, buf, 2); + } +} + +inline void msgpack_pack_signed_int_16(msgpack_pack_context x, int16_t d) +{ + const unsigned char buf[3] = {0xd1, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +inline void msgpack_pack_signed_int_32(msgpack_pack_context x, int32_t d) +{ + const unsigned char buf[5] = {0xd2, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +inline void msgpack_pack_signed_int_64(msgpack_pack_context x, int64_t d) +{ + // FIXME + const unsigned char buf[9] = {0xd3, STORE_64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + + +/* + * Float + */ + +inline void msgpack_pack_float(msgpack_pack_context x, float d) +{ + uint32_t n = *((uint32_t*)&d); // FIXME + const unsigned char buf[5] = {0xca, STORE_32(n)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +inline void msgpack_pack_double(msgpack_pack_context x, double d) +{ + uint64_t n = *((uint64_t*)&d); // FIXME + const unsigned char buf[9] = {0xcb, STORE_64(n)}; + msgpack_pack_append_buffer(x, buf, 9); +} + + +/* + * Nil + */ + +inline void msgpack_pack_nil(msgpack_pack_context x) +{ + static const unsigned char d = 0xc0; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Boolean + */ +inline void msgpack_pack_true(msgpack_pack_context x) +{ + static const unsigned char d = 0xc3; + msgpack_pack_append_buffer(x, &d, 1); +} + +inline void msgpack_pack_false(msgpack_pack_context x) +{ + static const unsigned char d = 0xc2; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Array + */ + +inline void msgpack_pack_array(msgpack_pack_context x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x90 | n; + msgpack_pack_append_buffer(x, &d, 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xdc, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdd, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Map + */ + +inline void msgpack_pack_map(msgpack_pack_context x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x80 | n; + msgpack_pack_append_buffer(x, &d, 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xde, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdf, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * String + */ + +inline void msgpack_pack_string(msgpack_pack_context x, const char* b) +{ + uint32_t l = strlen(b); + msgpack_pack_append_buffer(x, (const unsigned char*)b, l+1); +} + +inline void msgpack_pack_raw(msgpack_pack_context x, const void* b, size_t l) +{ + if(l < 32) { + unsigned char d = 0xa0 | l; + msgpack_pack_append_buffer(x, &d, 1); + } else if(l < 65536) { + uint16_t d = (uint16_t)l; + unsigned char buf[3] = {0xda, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)l; + unsigned char buf[5] = {0xdb, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } + msgpack_pack_append_buffer(x, b, l); +} + + +#endif /* msgpack/pack/inline_impl.h */ + diff --git a/msgpack/unpack/callback.h b/msgpack/unpack/callback.h new file mode 100644 index 0000000..b058a15 --- /dev/null +++ b/msgpack/unpack/callback.h @@ -0,0 +1,24 @@ + +msgpack_object msgpack_unpack_init(msgpack_unpack_context* x); +msgpack_object msgpack_unpack_unsigned_int_8(msgpack_unpack_context* x, uint8_t d); +msgpack_object msgpack_unpack_unsigned_int_16(msgpack_unpack_context* x, uint16_t d); +msgpack_object msgpack_unpack_unsigned_int_32(msgpack_unpack_context* x, uint32_t d); +msgpack_object msgpack_unpack_unsigned_int_64(msgpack_unpack_context* x, uint64_t d); +msgpack_object msgpack_unpack_signed_int_8(msgpack_unpack_context* x, int8_t d); +msgpack_object msgpack_unpack_signed_int_16(msgpack_unpack_context* x, int16_t d); +msgpack_object msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int32_t d); +msgpack_object msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d); +msgpack_object msgpack_unpack_float(msgpack_unpack_context* x, float d); +msgpack_object msgpack_unpack_double(msgpack_unpack_context* x, double d); +msgpack_object msgpack_unpack_big_int(msgpack_unpack_context* x, const void* b, unsigned int l); +msgpack_object msgpack_unpack_big_float(msgpack_unpack_context* x, const void* b, unsigned int l); +msgpack_object msgpack_unpack_nil(msgpack_unpack_context* x); +msgpack_object msgpack_unpack_true(msgpack_unpack_context* x); +msgpack_object msgpack_unpack_false(msgpack_unpack_context* x); +msgpack_object msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned int n); + void msgpack_unpack_array_item(msgpack_unpack_context* x, msgpack_object c, msgpack_object o); +msgpack_object msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n); + void msgpack_unpack_map_item(msgpack_unpack_context* x, msgpack_object c, msgpack_object k, msgpack_object v); +msgpack_object msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l); +msgpack_object msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, size_t l); + diff --git a/msgpack/unpack/inline_context.h b/msgpack/unpack/inline_context.h new file mode 100644 index 0000000..aecd566 --- /dev/null +++ b/msgpack/unpack/inline_context.h @@ -0,0 +1,52 @@ +/* + * MessagePack unpacking routine + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_UNPACK_INLINE_CONTEXT_H__ +#define MSGPACK_UNPACK_INLINE_CONTEXT_H__ + +#include +#include + +#ifndef MSG_STACK_SIZE +#define MSG_STACK_SIZE 16 +#endif + +typedef struct { + msgpack_object obj; + size_t count; + unsigned int ct; + union { + const unsigned char* terminal_trail_start; + msgpack_object map_key; + } tmp; +} msgpack_unpacker_stack; + +typedef struct { + msgpack_unpack_context user; // must be first + unsigned int cs; + size_t trail; + unsigned int top; + msgpack_unpacker_stack stack[MSG_STACK_SIZE]; +} msgpack_unpacker; + +void msgpack_unpacker_init(msgpack_unpacker* ctx); +int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len, size_t* off); +#define msgpack_unpacker_data(unpacker) (unpacker)->stack[0].obj + + +#endif /* msgpack/unpack/inline_context.h */ + diff --git a/msgpack/unpack/inline_impl.h b/msgpack/unpack/inline_impl.h new file mode 100644 index 0000000..ec7f0fc --- /dev/null +++ b/msgpack/unpack/inline_impl.h @@ -0,0 +1,438 @@ +/* + * MessagePack unpacking routine + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_UNPACK_INLINE_IMPL_H__ +#define MSGPACK_UNPACK_INLINE_IMPL_H__ + +#include +#include +#include +/*#include */ + +// Positive FixNum 0xxxxxxx 0x00 - 0x7f +// Negative FixNum 111xxxxx 0xe0 - 0xff +// Variable 110xxxxx 0xc0 - 0xdf +// nil 00000 0xc0 +// string 00001 0xc1 +// false 00010 0xc2 +// true 00011 0xc3 +// (?) 00100 0xc4 +// (?) 00101 0xc5 +// (?) 00110 0xc6 +// (?) 00111 0xc7 +// (?) 01000 0xc8 +// (?) 01001 0xc9 +// float 01010 0xca +// double 01011 0xcb +// uint 8 01100 0xcc +// uint 16 01101 0xcd +// uint 32 01110 0xce +// uint 64 01111 0xcf +// int 8 10000 0xd0 +// int 16 10001 0xd1 +// int 32 10010 0xd2 +// int 64 10011 0xd3 +// (?) 10100 0xd4 +// (?) 10101 0xd5 +// (big float 16) 10110 0xd6 +// (big float 32) 10111 0xd7 +// (big integer 16) 11000 0xd8 +// (big integer 32) 11001 0xd9 +// raw 16 11010 0xda +// raw 32 11011 0xdb +// array 16 11100 0xdc +// array 32 11101 0xdd +// map 16 11110 0xde +// map 32 11111 0xdf +// FixRaw 101xxxxx 0xa0 - 0xbf +// FixArray 1001xxxx 0x90 - 0x9f +// FixMap 1000xxxx 0x80 - 0x8f + + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +static inline uint64_t ntohll(uint64_t x) { +#ifdef __LITTLE_ENDIAN__ // FIXME +#if defined(__bswap_64) + return __bswap_64(x); +#elif defined(__DARWIN_OSSwapInt64) + return __DARWIN_OSSwapInt64(x); +#else + return ((x << 56) & 0xff00000000000000ULL ) | + ((x << 40) & 0x00ff000000000000ULL ) | + ((x << 24) & 0x0000ff0000000000ULL ) | + ((x << 8) & 0x000000ff00000000ULL ) | + ((x >> 8) & 0x00000000ff000000ULL ) | + ((x >> 24) & 0x0000000000ff0000ULL ) | + ((x >> 40) & 0x000000000000ff00ULL ) | + ((x >> 56) & 0x00000000000000ffULL ) ; +#endif +#else + return x; +#endif +} + +typedef enum { + CS_HEADER = 0x00, // nil + + CS_STRING = 0x01, + //CS_ = 0x02, // false + //CS_ = 0x03, // true + + //CS_ = 0x04, + //CS_ = 0x05, + //CS_ = 0x06, + //CS_ = 0x07, + + //CS_ = 0x08, + //CS_ = 0x09, + CS_FLOAT = 0x0a, + CS_DOUBLE = 0x0b, + CS_UNSIGNED_INT_8 = 0x0c, + CS_UNSIGNED_INT_16 = 0x0d, + CS_UNSIGNED_INT_32 = 0x0e, + CS_UNSIGNED_INT_64 = 0x0f, + CS_SIGNED_INT_8 = 0x10, + CS_SIGNED_INT_16 = 0x11, + CS_SIGNED_INT_32 = 0x12, + CS_SIGNED_INT_64 = 0x13, + + //CS_ = 0x14, + //CS_ = 0x15, + //CS_BIG_INT_16 = 0x16, + //CS_BIG_INT_32 = 0x17, + //CS_BIG_FLOAT_16 = 0x18, + //CS_BIG_FLOAT_32 = 0x19, + CS_RAW_16 = 0x1a, + CS_RAW_32 = 0x1b, + CS_ARRAY_16 = 0x1c, + CS_ARRAY_32 = 0x1d, + CS_MAP_16 = 0x1e, + CS_MAP_32 = 0x1f, + + //ACS_BIG_INT_VALUE, + //ACS_BIG_FLOAT_VALUE, + ACS_RAW_VALUE, +} current_state_t; + + +typedef enum { + CT_ARRAY_ITEM, + CT_MAP_KEY, + CT_MAP_VALUE, +} container_type_t; + + +void msgpack_unpacker_init(msgpack_unpacker* ctx) +{ + memset(ctx, 0, sizeof(msgpack_unpacker)); // FIXME init ctx->user? + ctx->cs = CS_HEADER; + ctx->trail = 0; + ctx->top = 0; + ctx->stack[0].obj = msgpack_unpack_init(&ctx->user); +} + +int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len, size_t* off) +{ + assert(len >= *off); + + const unsigned char* p = (unsigned char*)data + *off; + const unsigned char* const pe = (unsigned char*)data + len; + const void* n = NULL; + + size_t trail = ctx->trail; + unsigned int cs = ctx->cs; + unsigned int top = ctx->top; + msgpack_unpacker_stack* stack = ctx->stack; + msgpack_unpack_context* user = &ctx->user; + + msgpack_object obj; + + int ret; + +#define push_simple_value(func) \ + obj = func(user); \ + /*printf("obj %d\n",obj);*/ \ + goto _push +#define push_fixed_value(func, arg) \ + obj = func(user, arg); \ + /*printf("obj %d\n",obj);*/ \ + goto _push +#define push_variable_value(func, arg, arglen) \ + obj = func(user, arg, arglen); \ + /*printf("obj %d\n",obj);*/ \ + goto _push + +#define again_terminal_trail(_cs, from) \ + cs = _cs; \ + stack[top].tmp.terminal_trail_start = from; \ + goto _terminal_trail_again +#define again_fixed_trail(_cs, trail_len) \ + trail = trail_len; \ + cs = _cs; \ + goto _fixed_trail_again +#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ + trail = trail_len; \ + if(trail == 0) { goto ifzero; } \ + cs = _cs; \ + goto _fixed_trail_again + +#define start_container(func, count_, ct_) \ + stack[top].obj = func(user, count_); \ + if((count_) == 0) { obj = stack[top].obj; goto _push; } \ + if(top >= MSG_STACK_SIZE) { goto _failed; } \ + stack[top].ct = ct_; \ + stack[top].count = count_; \ + /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ + /*printf("stack push %d\n", top);*/ \ + ++top; \ + goto _header_again + +#define NEXT_CS(p) \ + ((unsigned int)*p & 0x1f) + +#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) +#define PTR_CAST_16(ptr) ntohs(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) ntohl(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) ntohll(*(uint64_t*)ptr) + + if(p == pe) { goto _out; } + do { + switch(cs) { + case CS_HEADER: + switch(*p) { + case 0x00 ... 0x7f: // Positive Fixnum + push_fixed_value(msgpack_unpack_unsigned_int_8, *(uint8_t*)p); + case 0xe0 ... 0xff: // Negative Fixnum + push_fixed_value(msgpack_unpack_signed_int_8, *(int8_t*)p); + case 0xc0 ... 0xdf: // Variable + switch(*p) { + case 0xc0: // nil + push_simple_value(msgpack_unpack_nil); + case 0xc1: // string + again_terminal_trail(NEXT_CS(p), p+1); + case 0xc2: // false + push_simple_value(msgpack_unpack_false); + case 0xc3: // true + push_simple_value(msgpack_unpack_true); + //case 0xc4: + //case 0xc5: + //case 0xc6: + //case 0xc7: + //case 0xc8: + //case 0xc9: + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); + //case 0xd4: + //case 0xd5: + //case 0xd6: // big integer 16 + //case 0xd7: // big integer 32 + //case 0xd8: // big float 16 + //case 0xd9: // big float 32 + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); + default: + goto _failed; + } + case 0xa0 ... 0xbf: // FixRaw + again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); + case 0x90 ... 0x9f: // FixArray + start_container(msgpack_unpack_array_start, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); + case 0x80 ... 0x8f: // FixMap + start_container(msgpack_unpack_map_start, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); + + default: + goto _failed; + } + // end CS_HEADER + + + _terminal_trail_again: + ++p; + + case CS_STRING: + if(*p == 0) { + const unsigned char* start = stack[top].tmp.terminal_trail_start; + obj = msgpack_unpack_string(user, start, p-start); + goto _push; + } + goto _terminal_trail_again; + + + _fixed_trail_again: + ++p; + + default: + if((size_t)(pe - p) < trail) { goto _out; } + n = p; p += trail - 1; + switch(cs) { + //case CS_ + //case CS_ + case CS_FLOAT: { + uint32_t x = PTR_CAST_32(n); // FIXME + push_fixed_value(msgpack_unpack_float, *((float*)&x)); } + case CS_DOUBLE: { + uint64_t x = PTR_CAST_64(n); // FIXME + push_fixed_value(msgpack_unpack_double, *((double*)&x)); } + case CS_UNSIGNED_INT_8: + push_fixed_value(msgpack_unpack_unsigned_int_8, (uint8_t)PTR_CAST_8(n)); + case CS_UNSIGNED_INT_16: + push_fixed_value(msgpack_unpack_unsigned_int_16, (uint16_t)PTR_CAST_16(n)); + case CS_UNSIGNED_INT_32: + push_fixed_value(msgpack_unpack_unsigned_int_32, (uint32_t)PTR_CAST_32(n)); + case CS_UNSIGNED_INT_64: + push_fixed_value(msgpack_unpack_unsigned_int_64, (uint64_t)PTR_CAST_64(n)); + + case CS_SIGNED_INT_8: + push_fixed_value(msgpack_unpack_signed_int_8, (int8_t)PTR_CAST_8(n)); + case CS_SIGNED_INT_16: + push_fixed_value(msgpack_unpack_signed_int_16, (int16_t)PTR_CAST_16(n)); + case CS_SIGNED_INT_32: + push_fixed_value(msgpack_unpack_signed_int_32, (int32_t)PTR_CAST_32(n)); + case CS_SIGNED_INT_64: + push_fixed_value(msgpack_unpack_signed_int_64, (int64_t)PTR_CAST_64(n)); + + //case CS_ + //case CS_ + //case CS_BIG_INT_16: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); + //case CS_BIG_INT_32: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); + //case ACS_BIG_INT_VALUE: + //_big_int_zero: + // // FIXME + // push_variable_value(msgpack_unpack_big_int, n, trail); + + //case CS_BIG_FLOAT_16: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); + //case CS_BIG_FLOAT_32: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); + //case ACS_BIG_FLOAT_VALUE: + //_big_float_zero: + // // FIXME + // push_variable_value(msgpack_unpack_big_float, n, trail); + + case CS_RAW_16: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); + case CS_RAW_32: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); + case ACS_RAW_VALUE: + _raw_zero: + push_variable_value(msgpack_unpack_raw, n, trail); + + case CS_ARRAY_16: + start_container(msgpack_unpack_array_start, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + case CS_ARRAY_32: + start_container(msgpack_unpack_array_start, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + + case CS_MAP_16: + start_container(msgpack_unpack_map_start, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + case CS_MAP_32: + start_container(msgpack_unpack_map_start, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + + default: + goto _failed; + } + } + +_push: + if(top == 0) { goto _finish; } + msgpack_unpacker_stack* c = &stack[top-1]; + switch(c->ct) { + case CT_ARRAY_ITEM: + msgpack_unpack_array_item(user, c->obj, obj); + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + goto _header_again; + case CT_MAP_KEY: + c->tmp.map_key = obj; + c->ct = CT_MAP_VALUE; + goto _header_again; + case CT_MAP_VALUE: + msgpack_unpack_map_item(user, c->obj, c->tmp.map_key, obj); + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + c->ct = CT_MAP_KEY; + goto _header_again; + + default: + goto _failed; + } + +_header_again: + cs = CS_HEADER; + ++p; + } while(p != pe); + goto _out; + + +_finish: + stack[0].obj = obj; + ++p; + ret = 1; + /*printf("-- finish --\n"); */ + goto _end; + +_failed: + /*printf("** FAILED **\n"); */ + ret = -1; + goto _end; + +_out: + ret = 0; + goto _end; + +_end: + ctx->cs = cs; + ctx->trail = trail; + ctx->top = top; + *off = p - (const unsigned char*)data; + + return ret; +} + + +#endif /* msgpack/unpack/inline_impl.h */ + diff --git a/ruby/Makefile b/ruby/Makefile new file mode 100644 index 0000000..6ef04b0 --- /dev/null +++ b/ruby/Makefile @@ -0,0 +1,153 @@ + +SHELL = /bin/sh + +#### Start of system configuration section. #### + +srcdir = . +topdir = /Users/frsyuki/ports/lib/ruby/1.8/i686-darwin9.1.0 +hdrdir = $(topdir) +VPATH = $(srcdir):$(topdir):$(hdrdir) +prefix = $(DESTDIR)/Users/frsyuki/ports +exec_prefix = $(prefix) +sitedir = $(prefix)/lib/ruby/site_ruby +rubylibdir = $(libdir)/ruby/$(ruby_version) +docdir = $(datarootdir)/doc/$(PACKAGE) +dvidir = $(docdir) +datarootdir = $(prefix)/share +archdir = $(rubylibdir)/$(arch) +sbindir = $(exec_prefix)/sbin +psdir = $(docdir) +vendordir = $(prefix)/lib/ruby/vendor_ruby +localedir = $(datarootdir)/locale +htmldir = $(docdir) +datadir = $(datarootdir) +includedir = $(prefix)/include +infodir = $(datarootdir)/info +sysconfdir = $(prefix)/etc +mandir = $(DESTDIR)/Users/frsyuki/ports/share/man +libdir = $(exec_prefix)/lib +sharedstatedir = $(prefix)/com +oldincludedir = $(DESTDIR)/usr/include +pdfdir = $(docdir) +sitearchdir = $(sitelibdir)/$(sitearch) +vendorarchdir = $(vendorlibdir)/$(vendorarch) +bindir = $(exec_prefix)/bin +localstatedir = $(prefix)/var +vendorlibdir = $(vendordir)/$(ruby_version) +sitelibdir = $(sitedir)/$(ruby_version) +libexecdir = $(exec_prefix)/libexec + +CC = /usr/bin/gcc-4.0 +LIBRUBY = $(LIBRUBY_SO) +LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a +LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME) +LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static + +RUBY_EXTCONF_H = +CFLAGS = -fno-common -O2 -fno-common -pipe -fno-common -I.. -Wall -O9 +INCFLAGS = -I. -I$(topdir) -I$(hdrdir) -I$(srcdir) +CPPFLAGS = -I/Users/frsyuki/ports/include +CXXFLAGS = $(CFLAGS) +DLDFLAGS = -L. -L/Users/frsyuki/ports/lib +LDSHARED = cc -dynamic -bundle -undefined suppress -flat_namespace +AR = ar +EXEEXT = + +RUBY_INSTALL_NAME = ruby +RUBY_SO_NAME = ruby +arch = i686-darwin9.1.0 +sitearch = i686-darwin9.1.0 +vendorarch = i686-darwin9.1.0 +ruby_version = 1.8 +ruby = /Users/frsyuki/ports/bin/ruby +RUBY = $(ruby) +RM = rm -f +MAKEDIRS = mkdir -p +INSTALL = /usr/bin/install +INSTALL_PROG = $(INSTALL) -m 0755 +INSTALL_DATA = $(INSTALL) -m 644 +COPY = cp + +#### End of system configuration section. #### + +preload = + +libpath = . $(libdir) +LIBPATH = -L"." -L"$(libdir)" +DEFFILE = + +CLEANFILES = mkmf.log +DISTCLEANFILES = + +extout = +extout_prefix = +target_prefix = +LOCAL_LIBS = +LIBS = $(LIBRUBYARG_SHARED) -lpthread -ldl -lobjc +SRCS = pack.c rbinit.c unpack.c unpack_inline.c +OBJS = pack.o rbinit.o unpack.o unpack_inline.o +TARGET = msgpack +DLLIB = $(TARGET).bundle +EXTSTATIC = +STATIC_LIB = + +RUBYCOMMONDIR = $(sitedir)$(target_prefix) +RUBYLIBDIR = $(sitelibdir)$(target_prefix) +RUBYARCHDIR = $(sitearchdir)$(target_prefix) + +TARGET_SO = $(DLLIB) +CLEANLIBS = $(TARGET).bundle $(TARGET).il? $(TARGET).tds $(TARGET).map +CLEANOBJS = *.o *.a *.s[ol] *.pdb *.exp *.bak + +all: $(DLLIB) +static: $(STATIC_LIB) + +clean: + @-$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) + +distclean: clean + @-$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log + @-$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES) + +realclean: distclean +install: install-so install-rb + +install-so: $(RUBYARCHDIR) +install-so: $(RUBYARCHDIR)/$(DLLIB) +$(RUBYARCHDIR)/$(DLLIB): $(DLLIB) + $(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR) +install-rb: pre-install-rb install-rb-default +install-rb-default: pre-install-rb-default +pre-install-rb: Makefile +pre-install-rb-default: Makefile +$(RUBYARCHDIR): + $(MAKEDIRS) $@ + +site-install: site-install-so site-install-rb +site-install-so: install-so +site-install-rb: install-rb + +.SUFFIXES: .c .m .cc .cxx .cpp .C .o + +.cc.o: + $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< + +.cxx.o: + $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< + +.cpp.o: + $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< + +.C.o: + $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< + +.c.o: + $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) -c $< + +$(DLLIB): $(OBJS) + @-$(RM) $@ + $(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS) + + + +$(OBJS): ruby.h defines.h diff --git a/ruby/bench.rb b/ruby/bench.rb new file mode 100644 index 0000000..1e6e27b --- /dev/null +++ b/ruby/bench.rb @@ -0,0 +1,60 @@ +require 'rubygems' +require 'json' +require 'msgpack' + +def show10(str) + puts "#{str.length/1024} KB" + puts str[0, 10].unpack('C*').map{|x|"%02x"%x}.join(' ') + " ..." +end + +ary = [] +i = 0 +while i < (1<<23) + ary << (1<<23) + #ary << i + i += 1 +end + +GC.start + +puts "----" +puts "MessagePack" +a = Time.now +packed = MessagePack::pack(ary) +b = Time.now +show10(packed) +puts "#{b-a} sec." + +GC.start + +puts "----" +puts "JSON" +a = Time.now +json = ary.to_json +b = Time.now +show10(json) +puts "#{b-a} sec." + +ary = nil +GC.start + + +puts "----" +puts "MessagePack" +a = Time.now +ary = MessagePack::unpack(packed) +b = Time.now +puts "#{b-a} sec." + +ary = nil +GC.start + + +puts "----" +puts "JSON" +a = Time.now +ary = JSON::load(json) +b = Time.now +puts "#{b-a} sec." + + diff --git a/ruby/extconf.rb b/ruby/extconf.rb new file mode 100644 index 0000000..88abb55 --- /dev/null +++ b/ruby/extconf.rb @@ -0,0 +1,4 @@ +require 'mkmf' +$CFLAGS << " -I.. -Wall -O9" +create_makefile('msgpack') + diff --git a/ruby/gem.sh b/ruby/gem.sh new file mode 100755 index 0000000..9d9f429 --- /dev/null +++ b/ruby/gem.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +cp extconf.rb gem/ext/ +cp pack.c gem/ext/ +cp pack.h gem/ext/ +cp pack_inline.h gem/ext/ +cp rbinit.c gem/ext/ +cp unpack.c gem/ext/ +cp unpack.h gem/ext/ +cp unpack_context.h gem/ext/ +cp unpack_inline.c gem/ext/ +cp ../README gem/README.txt +cp ../msgpack/pack/inline_context.h gem/msgpack/pack/ +cp ../msgpack/pack/inline_impl.h gem/msgpack/pack/ +cp ../msgpack/unpack/inline_context.h gem/msgpack/unpack/ +cp ../msgpack/unpack/inline_impl.h gem/msgpack/unpack/ + +cd gem && rake --trace package + diff --git a/ruby/gem/AUTHORS b/ruby/gem/AUTHORS new file mode 100644 index 0000000..ababacb --- /dev/null +++ b/ruby/gem/AUTHORS @@ -0,0 +1 @@ +FURUHASHI Sadayuki diff --git a/ruby/gem/History.txt b/ruby/gem/History.txt new file mode 100644 index 0000000..e69de29 diff --git a/ruby/gem/License.txt b/ruby/gem/License.txt new file mode 100644 index 0000000..f4000b7 --- /dev/null +++ b/ruby/gem/License.txt @@ -0,0 +1,13 @@ +Copyright 2008 Furuhashi Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/ruby/gem/Manifest.txt b/ruby/gem/Manifest.txt new file mode 100644 index 0000000..fac8555 --- /dev/null +++ b/ruby/gem/Manifest.txt @@ -0,0 +1,26 @@ +License.txt +Manifest.txt +README.txt +Rakefile +config/hoe.rb +config/requirements.rb +ext/extconf.rb +ext/pack.c +ext/pack.h +ext/pack_inline.h +ext/rbinit.c +ext/unpack.c +ext/unpack.h +ext/unpack_context.h +ext/unpack_inline.c +msgpack/pack/inline_context.h +msgpack/pack/inline_impl.h +msgpack/unpack/inline_context.h +msgpack/unpack/inline_impl.h +lib/msgpack/version.rb +script/console +script/destroy +script/generate +setup.rb +tasks/deployment.rake +tasks/environment.rake diff --git a/ruby/gem/PostInstall.txt b/ruby/gem/PostInstall.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/ruby/gem/PostInstall.txt @@ -0,0 +1 @@ + diff --git a/ruby/gem/Rakefile b/ruby/gem/Rakefile new file mode 100644 index 0000000..e469154 --- /dev/null +++ b/ruby/gem/Rakefile @@ -0,0 +1,4 @@ +require 'config/requirements' +require 'config/hoe' # setup Hoe + all gem configuration + +Dir['tasks/**/*.rake'].each { |rake| load rake } \ No newline at end of file diff --git a/ruby/gem/config/hoe.rb b/ruby/gem/config/hoe.rb new file mode 100644 index 0000000..8500acf --- /dev/null +++ b/ruby/gem/config/hoe.rb @@ -0,0 +1,75 @@ +require 'msgpack/version' + +AUTHOR = 'FURUHASHI Sadayuki' # can also be an array of Authors +EMAIL = "fr _at_ syuki.skr.jp" +DESCRIPTION = "An object-oriented parser generator based on Parser Expression Grammar" +GEM_NAME = 'msgpack' # what ppl will type to install your gem +RUBYFORGE_PROJECT = 'msgpack' # The unix name for your project +HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org" +DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}" +EXTRA_DEPENDENCIES = [ +# ['activesupport', '>= 1.3.1'] +] # An array of rubygem dependencies [name, version] + +@config_file = "~/.rubyforge/user-config.yml" +@config = nil +RUBYFORGE_USERNAME = "unknown" +def rubyforge_username + unless @config + begin + @config = YAML.load(File.read(File.expand_path(@config_file))) + rescue + puts <<-EOS +ERROR: No rubyforge config file found: #{@config_file} +Run 'rubyforge setup' to prepare your env for access to Rubyforge + - See http://newgem.rubyforge.org/rubyforge.html for more details + EOS + exit + end + end + RUBYFORGE_USERNAME.replace @config["username"] +end + + +REV = nil +# UNCOMMENT IF REQUIRED: +# REV = YAML.load(`svn info`)['Revision'] +VERS = MessagePack::VERSION::STRING + (REV ? ".#{REV}" : "") +RDOC_OPTS = ['--quiet', '--title', 'msgpack documentation', + "--opname", "index.html", + "--line-numbers", + "--main", "README", + "--inline-source"] + +class Hoe + def extra_deps + @extra_deps.reject! { |x| Array(x).first == 'hoe' } + @extra_deps + end +end + +# Generate all the Rake tasks +# Run 'rake -T' to see list of generated tasks (from gem root directory) +$hoe = Hoe.new(GEM_NAME, VERS) do |p| + p.developer(AUTHOR, EMAIL) + p.description = DESCRIPTION + p.summary = DESCRIPTION + p.url = HOMEPATH + p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT + p.test_globs = ["test/**/test_*.rb"] + p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store'] #An array of file patterns to delete on clean. + + # == Optional + p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n") + #p.extra_deps = EXTRA_DEPENDENCIES + + p.spec_extras = { # A hash of extra values to set in the gemspec. + :extensions => %w[ext/extconf.rb] + } +end + +CHANGES = $hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n") +PATH = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}" +$hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc') +$hoe.rsync_args = '-av --delete --ignore-errors' +$hoe.spec.post_install_message = File.open(File.dirname(__FILE__) + "/../PostInstall.txt").read rescue "" diff --git a/ruby/gem/config/requirements.rb b/ruby/gem/config/requirements.rb new file mode 100644 index 0000000..9292b69 --- /dev/null +++ b/ruby/gem/config/requirements.rb @@ -0,0 +1,15 @@ +require 'fileutils' +include FileUtils + +require 'rubygems' +%w[rake hoe newgem rubigen].each do |req_gem| + begin + require req_gem + rescue LoadError + puts "This Rakefile requires the '#{req_gem}' RubyGem." + puts "Installation: gem install #{req_gem} -y" + exit + end +end + +$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib])) diff --git a/ruby/gem/lib/msgpack/version.rb b/ruby/gem/lib/msgpack/version.rb new file mode 100644 index 0000000..44d1dc0 --- /dev/null +++ b/ruby/gem/lib/msgpack/version.rb @@ -0,0 +1,9 @@ +module MessagePack + module VERSION #:nodoc: + MAJOR = 0 + MINOR = 0 + TINY = 1 + + STRING = [MAJOR, MINOR, TINY].join('.') + end +end diff --git a/ruby/gem/script/console b/ruby/gem/script/console new file mode 100755 index 0000000..76f32a0 --- /dev/null +++ b/ruby/gem/script/console @@ -0,0 +1,10 @@ +#!/usr/bin/env ruby +# File: script/console +irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb' + +libs = " -r irb/completion" +# Perhaps use a console_lib to store any extra methods I may want available in the cosole +# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}" +libs << " -r #{File.dirname(__FILE__) + '/../lib/msgpack.rb'}" +puts "Loading msgpack gem" +exec "#{irb} #{libs} --simple-prompt" diff --git a/ruby/gem/script/destroy b/ruby/gem/script/destroy new file mode 100755 index 0000000..e48464d --- /dev/null +++ b/ruby/gem/script/destroy @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby +APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..')) + +begin + require 'rubigen' +rescue LoadError + require 'rubygems' + require 'rubigen' +end +require 'rubigen/scripts/destroy' + +ARGV.shift if ['--help', '-h'].include?(ARGV[0]) +RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit] +RubiGen::Scripts::Destroy.new.run(ARGV) diff --git a/ruby/gem/script/generate b/ruby/gem/script/generate new file mode 100755 index 0000000..c27f655 --- /dev/null +++ b/ruby/gem/script/generate @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby +APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..')) + +begin + require 'rubigen' +rescue LoadError + require 'rubygems' + require 'rubigen' +end +require 'rubigen/scripts/generate' + +ARGV.shift if ['--help', '-h'].include?(ARGV[0]) +RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit] +RubiGen::Scripts::Generate.new.run(ARGV) diff --git a/ruby/gem/script/txt2html b/ruby/gem/script/txt2html new file mode 100755 index 0000000..09c583f --- /dev/null +++ b/ruby/gem/script/txt2html @@ -0,0 +1,82 @@ +#!/usr/bin/env ruby + +GEM_NAME = 'msgpack' # what ppl will type to install your gem +RUBYFORGE_PROJECT = 'msgpack' + +require 'rubygems' +begin + require 'newgem' + require 'rubyforge' +rescue LoadError + puts "\n\nGenerating the website requires the newgem RubyGem" + puts "Install: gem install newgem\n\n" + exit(1) +end +require 'redcloth' +require 'syntax/convertors/html' +require 'erb' +require File.dirname(__FILE__) + "/../lib/#{GEM_NAME}/version.rb" + +version = MessagePack::VERSION::STRING +download = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}" + +def rubyforge_project_id + RubyForge.new.autoconfig["group_ids"][RUBYFORGE_PROJECT] +end + +class Fixnum + def ordinal + # teens + return 'th' if (10..19).include?(self % 100) + # others + case self % 10 + when 1: return 'st' + when 2: return 'nd' + when 3: return 'rd' + else return 'th' + end + end +end + +class Time + def pretty + return "#{mday}#{mday.ordinal} #{strftime('%B')} #{year}" + end +end + +def convert_syntax(syntax, source) + return Syntax::Convertors::HTML.for_syntax(syntax).convert(source).gsub(%r!^
|
$!,'') +end + +if ARGV.length >= 1 + src, template = ARGV + template ||= File.join(File.dirname(__FILE__), '/../website/template.html.erb') +else + puts("Usage: #{File.split($0).last} source.txt [template.html.erb] > output.html") + exit! +end + +template = ERB.new(File.open(template).read) + +title = nil +body = nil +File.open(src) do |fsrc| + title_text = fsrc.readline + body_text_template = fsrc.read + body_text = ERB.new(body_text_template).result(binding) + syntax_items = [] + body_text.gsub!(%r!<(pre|code)[^>]*?syntax=['"]([^'"]+)[^>]*>(.*?)!m){ + ident = syntax_items.length + element, syntax, source = $1, $2, $3 + syntax_items << "<#{element} class='syntax'>#{convert_syntax(syntax, source)}" + "syntax-temp-#{ident}" + } + title = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip + body = RedCloth.new(body_text).to_html + body.gsub!(%r!(?:
)?syntax-temp-(\d+)(?:
)?!){ syntax_items[$1.to_i] } +end +stat = File.stat(src) +created = stat.ctime +modified = stat.mtime + +$stdout << template.result(binding) diff --git a/ruby/gem/setup.rb b/ruby/gem/setup.rb new file mode 100644 index 0000000..424a5f3 --- /dev/null +++ b/ruby/gem/setup.rb @@ -0,0 +1,1585 @@ +# +# setup.rb +# +# Copyright (c) 2000-2005 Minero Aoki +# +# This program is free software. +# You can distribute/modify this program under the terms of +# the GNU LGPL, Lesser General Public License version 2.1. +# + +unless Enumerable.method_defined?(:map) # Ruby 1.4.6 + module Enumerable + alias map collect + end +end + +unless File.respond_to?(:read) # Ruby 1.6 + def File.read(fname) + open(fname) {|f| + return f.read + } + end +end + +unless Errno.const_defined?(:ENOTEMPTY) # Windows? + module Errno + class ENOTEMPTY + # We do not raise this exception, implementation is not needed. + end + end +end + +def File.binread(fname) + open(fname, 'rb') {|f| + return f.read + } +end + +# for corrupted Windows' stat(2) +def File.dir?(path) + File.directory?((path[-1,1] == '/') ? path : path + '/') +end + + +class ConfigTable + + include Enumerable + + def initialize(rbconfig) + @rbconfig = rbconfig + @items = [] + @table = {} + # options + @install_prefix = nil + @config_opt = nil + @verbose = true + @no_harm = false + end + + attr_accessor :install_prefix + attr_accessor :config_opt + + attr_writer :verbose + + def verbose? + @verbose + end + + attr_writer :no_harm + + def no_harm? + @no_harm + end + + def [](key) + lookup(key).resolve(self) + end + + def []=(key, val) + lookup(key).set val + end + + def names + @items.map {|i| i.name } + end + + def each(&block) + @items.each(&block) + end + + def key?(name) + @table.key?(name) + end + + def lookup(name) + @table[name] or setup_rb_error "no such config item: #{name}" + end + + def add(item) + @items.push item + @table[item.name] = item + end + + def remove(name) + item = lookup(name) + @items.delete_if {|i| i.name == name } + @table.delete_if {|name, i| i.name == name } + item + end + + def load_script(path, inst = nil) + if File.file?(path) + MetaConfigEnvironment.new(self, inst).instance_eval File.read(path), path + end + end + + def savefile + '.config' + end + + def load_savefile + begin + File.foreach(savefile()) do |line| + k, v = *line.split(/=/, 2) + self[k] = v.strip + end + rescue Errno::ENOENT + setup_rb_error $!.message + "\n#{File.basename($0)} config first" + end + end + + def save + @items.each {|i| i.value } + File.open(savefile(), 'w') {|f| + @items.each do |i| + f.printf "%s=%s\n", i.name, i.value if i.value? and i.value + end + } + end + + def load_standard_entries + standard_entries(@rbconfig).each do |ent| + add ent + end + end + + def standard_entries(rbconfig) + c = rbconfig + + rubypath = File.join(c['bindir'], c['ruby_install_name'] + c['EXEEXT']) + + major = c['MAJOR'].to_i + minor = c['MINOR'].to_i + teeny = c['TEENY'].to_i + version = "#{major}.#{minor}" + + # ruby ver. >= 1.4.4? + newpath_p = ((major >= 2) or + ((major == 1) and + ((minor >= 5) or + ((minor == 4) and (teeny >= 4))))) + + if c['rubylibdir'] + # V > 1.6.3 + libruby = "#{c['prefix']}/lib/ruby" + librubyver = c['rubylibdir'] + librubyverarch = c['archdir'] + siteruby = c['sitedir'] + siterubyver = c['sitelibdir'] + siterubyverarch = c['sitearchdir'] + elsif newpath_p + # 1.4.4 <= V <= 1.6.3 + libruby = "#{c['prefix']}/lib/ruby" + librubyver = "#{c['prefix']}/lib/ruby/#{version}" + librubyverarch = "#{c['prefix']}/lib/ruby/#{version}/#{c['arch']}" + siteruby = c['sitedir'] + siterubyver = "$siteruby/#{version}" + siterubyverarch = "$siterubyver/#{c['arch']}" + else + # V < 1.4.4 + libruby = "#{c['prefix']}/lib/ruby" + librubyver = "#{c['prefix']}/lib/ruby/#{version}" + librubyverarch = "#{c['prefix']}/lib/ruby/#{version}/#{c['arch']}" + siteruby = "#{c['prefix']}/lib/ruby/#{version}/site_ruby" + siterubyver = siteruby + siterubyverarch = "$siterubyver/#{c['arch']}" + end + parameterize = lambda {|path| + path.sub(/\A#{Regexp.quote(c['prefix'])}/, '$prefix') + } + + if arg = c['configure_args'].split.detect {|arg| /--with-make-prog=/ =~ arg } + makeprog = arg.sub(/'/, '').split(/=/, 2)[1] + else + makeprog = 'make' + end + + [ + ExecItem.new('installdirs', 'std/site/home', + 'std: install under libruby; site: install under site_ruby; home: install under $HOME')\ + {|val, table| + case val + when 'std' + table['rbdir'] = '$librubyver' + table['sodir'] = '$librubyverarch' + when 'site' + table['rbdir'] = '$siterubyver' + table['sodir'] = '$siterubyverarch' + when 'home' + setup_rb_error '$HOME was not set' unless ENV['HOME'] + table['prefix'] = ENV['HOME'] + table['rbdir'] = '$libdir/ruby' + table['sodir'] = '$libdir/ruby' + end + }, + PathItem.new('prefix', 'path', c['prefix'], + 'path prefix of target environment'), + PathItem.new('bindir', 'path', parameterize.call(c['bindir']), + 'the directory for commands'), + PathItem.new('libdir', 'path', parameterize.call(c['libdir']), + 'the directory for libraries'), + PathItem.new('datadir', 'path', parameterize.call(c['datadir']), + 'the directory for shared data'), + PathItem.new('mandir', 'path', parameterize.call(c['mandir']), + 'the directory for man pages'), + PathItem.new('sysconfdir', 'path', parameterize.call(c['sysconfdir']), + 'the directory for system configuration files'), + PathItem.new('localstatedir', 'path', parameterize.call(c['localstatedir']), + 'the directory for local state data'), + PathItem.new('libruby', 'path', libruby, + 'the directory for ruby libraries'), + PathItem.new('librubyver', 'path', librubyver, + 'the directory for standard ruby libraries'), + PathItem.new('librubyverarch', 'path', librubyverarch, + 'the directory for standard ruby extensions'), + PathItem.new('siteruby', 'path', siteruby, + 'the directory for version-independent aux ruby libraries'), + PathItem.new('siterubyver', 'path', siterubyver, + 'the directory for aux ruby libraries'), + PathItem.new('siterubyverarch', 'path', siterubyverarch, + 'the directory for aux ruby binaries'), + PathItem.new('rbdir', 'path', '$siterubyver', + 'the directory for ruby scripts'), + PathItem.new('sodir', 'path', '$siterubyverarch', + 'the directory for ruby extentions'), + PathItem.new('rubypath', 'path', rubypath, + 'the path to set to #! line'), + ProgramItem.new('rubyprog', 'name', rubypath, + 'the ruby program using for installation'), + ProgramItem.new('makeprog', 'name', makeprog, + 'the make program to compile ruby extentions'), + SelectItem.new('shebang', 'all/ruby/never', 'ruby', + 'shebang line (#!) editing mode'), + BoolItem.new('without-ext', 'yes/no', 'no', + 'does not compile/install ruby extentions') + ] + end + private :standard_entries + + def load_multipackage_entries + multipackage_entries().each do |ent| + add ent + end + end + + def multipackage_entries + [ + PackageSelectionItem.new('with', 'name,name...', '', 'ALL', + 'package names that you want to install'), + PackageSelectionItem.new('without', 'name,name...', '', 'NONE', + 'package names that you do not want to install') + ] + end + private :multipackage_entries + + ALIASES = { + 'std-ruby' => 'librubyver', + 'stdruby' => 'librubyver', + 'rubylibdir' => 'librubyver', + 'archdir' => 'librubyverarch', + 'site-ruby-common' => 'siteruby', # For backward compatibility + 'site-ruby' => 'siterubyver', # For backward compatibility + 'bin-dir' => 'bindir', + 'bin-dir' => 'bindir', + 'rb-dir' => 'rbdir', + 'so-dir' => 'sodir', + 'data-dir' => 'datadir', + 'ruby-path' => 'rubypath', + 'ruby-prog' => 'rubyprog', + 'ruby' => 'rubyprog', + 'make-prog' => 'makeprog', + 'make' => 'makeprog' + } + + def fixup + ALIASES.each do |ali, name| + @table[ali] = @table[name] + end + @items.freeze + @table.freeze + @options_re = /\A--(#{@table.keys.join('|')})(?:=(.*))?\z/ + end + + def parse_opt(opt) + m = @options_re.match(opt) or setup_rb_error "config: unknown option #{opt}" + m.to_a[1,2] + end + + def dllext + @rbconfig['DLEXT'] + end + + def value_config?(name) + lookup(name).value? + end + + class Item + def initialize(name, template, default, desc) + @name = name.freeze + @template = template + @value = default + @default = default + @description = desc + end + + attr_reader :name + attr_reader :description + + attr_accessor :default + alias help_default default + + def help_opt + "--#{@name}=#{@template}" + end + + def value? + true + end + + def value + @value + end + + def resolve(table) + @value.gsub(%r<\$([^/]+)>) { table[$1] } + end + + def set(val) + @value = check(val) + end + + private + + def check(val) + setup_rb_error "config: --#{name} requires argument" unless val + val + end + end + + class BoolItem < Item + def config_type + 'bool' + end + + def help_opt + "--#{@name}" + end + + private + + def check(val) + return 'yes' unless val + case val + when /\Ay(es)?\z/i, /\At(rue)?\z/i then 'yes' + when /\An(o)?\z/i, /\Af(alse)\z/i then 'no' + else + setup_rb_error "config: --#{@name} accepts only yes/no for argument" + end + end + end + + class PathItem < Item + def config_type + 'path' + end + + private + + def check(path) + setup_rb_error "config: --#{@name} requires argument" unless path + path[0,1] == '$' ? path : File.expand_path(path) + end + end + + class ProgramItem < Item + def config_type + 'program' + end + end + + class SelectItem < Item + def initialize(name, selection, default, desc) + super + @ok = selection.split('/') + end + + def config_type + 'select' + end + + private + + def check(val) + unless @ok.include?(val.strip) + setup_rb_error "config: use --#{@name}=#{@template} (#{val})" + end + val.strip + end + end + + class ExecItem < Item + def initialize(name, selection, desc, &block) + super name, selection, nil, desc + @ok = selection.split('/') + @action = block + end + + def config_type + 'exec' + end + + def value? + false + end + + def resolve(table) + setup_rb_error "$#{name()} wrongly used as option value" + end + + undef set + + def evaluate(val, table) + v = val.strip.downcase + unless @ok.include?(v) + setup_rb_error "invalid option --#{@name}=#{val} (use #{@template})" + end + @action.call v, table + end + end + + class PackageSelectionItem < Item + def initialize(name, template, default, help_default, desc) + super name, template, default, desc + @help_default = help_default + end + + attr_reader :help_default + + def config_type + 'package' + end + + private + + def check(val) + unless File.dir?("packages/#{val}") + setup_rb_error "config: no such package: #{val}" + end + val + end + end + + class MetaConfigEnvironment + def initialize(config, installer) + @config = config + @installer = installer + end + + def config_names + @config.names + end + + def config?(name) + @config.key?(name) + end + + def bool_config?(name) + @config.lookup(name).config_type == 'bool' + end + + def path_config?(name) + @config.lookup(name).config_type == 'path' + end + + def value_config?(name) + @config.lookup(name).config_type != 'exec' + end + + def add_config(item) + @config.add item + end + + def add_bool_config(name, default, desc) + @config.add BoolItem.new(name, 'yes/no', default ? 'yes' : 'no', desc) + end + + def add_path_config(name, default, desc) + @config.add PathItem.new(name, 'path', default, desc) + end + + def set_config_default(name, default) + @config.lookup(name).default = default + end + + def remove_config(name) + @config.remove(name) + end + + # For only multipackage + def packages + raise '[setup.rb fatal] multi-package metaconfig API packages() called for single-package; contact application package vendor' unless @installer + @installer.packages + end + + # For only multipackage + def declare_packages(list) + raise '[setup.rb fatal] multi-package metaconfig API declare_packages() called for single-package; contact application package vendor' unless @installer + @installer.packages = list + end + end + +end # class ConfigTable + + +# This module requires: #verbose?, #no_harm? +module FileOperations + + def mkdir_p(dirname, prefix = nil) + dirname = prefix + File.expand_path(dirname) if prefix + $stderr.puts "mkdir -p #{dirname}" if verbose? + return if no_harm? + + # Does not check '/', it's too abnormal. + dirs = File.expand_path(dirname).split(%r<(?=/)>) + if /\A[a-z]:\z/i =~ dirs[0] + disk = dirs.shift + dirs[0] = disk + dirs[0] + end + dirs.each_index do |idx| + path = dirs[0..idx].join('') + Dir.mkdir path unless File.dir?(path) + end + end + + def rm_f(path) + $stderr.puts "rm -f #{path}" if verbose? + return if no_harm? + force_remove_file path + end + + def rm_rf(path) + $stderr.puts "rm -rf #{path}" if verbose? + return if no_harm? + remove_tree path + end + + def remove_tree(path) + if File.symlink?(path) + remove_file path + elsif File.dir?(path) + remove_tree0 path + else + force_remove_file path + end + end + + def remove_tree0(path) + Dir.foreach(path) do |ent| + next if ent == '.' + next if ent == '..' + entpath = "#{path}/#{ent}" + if File.symlink?(entpath) + remove_file entpath + elsif File.dir?(entpath) + remove_tree0 entpath + else + force_remove_file entpath + end + end + begin + Dir.rmdir path + rescue Errno::ENOTEMPTY + # directory may not be empty + end + end + + def move_file(src, dest) + force_remove_file dest + begin + File.rename src, dest + rescue + File.open(dest, 'wb') {|f| + f.write File.binread(src) + } + File.chmod File.stat(src).mode, dest + File.unlink src + end + end + + def force_remove_file(path) + begin + remove_file path + rescue + end + end + + def remove_file(path) + File.chmod 0777, path + File.unlink path + end + + def install(from, dest, mode, prefix = nil) + $stderr.puts "install #{from} #{dest}" if verbose? + return if no_harm? + + realdest = prefix ? prefix + File.expand_path(dest) : dest + realdest = File.join(realdest, File.basename(from)) if File.dir?(realdest) + str = File.binread(from) + if diff?(str, realdest) + verbose_off { + rm_f realdest if File.exist?(realdest) + } + File.open(realdest, 'wb') {|f| + f.write str + } + File.chmod mode, realdest + + File.open("#{objdir_root()}/InstalledFiles", 'a') {|f| + if prefix + f.puts realdest.sub(prefix, '') + else + f.puts realdest + end + } + end + end + + def diff?(new_content, path) + return true unless File.exist?(path) + new_content != File.binread(path) + end + + def command(*args) + $stderr.puts args.join(' ') if verbose? + system(*args) or raise RuntimeError, + "system(#{args.map{|a| a.inspect }.join(' ')}) failed" + end + + def ruby(*args) + command config('rubyprog'), *args + end + + def make(task = nil) + command(*[config('makeprog'), task].compact) + end + + def extdir?(dir) + File.exist?("#{dir}/MANIFEST") or File.exist?("#{dir}/extconf.rb") + end + + def files_of(dir) + Dir.open(dir) {|d| + return d.select {|ent| File.file?("#{dir}/#{ent}") } + } + end + + DIR_REJECT = %w( . .. CVS SCCS RCS CVS.adm .svn ) + + def directories_of(dir) + Dir.open(dir) {|d| + return d.select {|ent| File.dir?("#{dir}/#{ent}") } - DIR_REJECT + } + end + +end + + +# This module requires: #srcdir_root, #objdir_root, #relpath +module HookScriptAPI + + def get_config(key) + @config[key] + end + + alias config get_config + + # obsolete: use metaconfig to change configuration + def set_config(key, val) + @config[key] = val + end + + # + # srcdir/objdir (works only in the package directory) + # + + def curr_srcdir + "#{srcdir_root()}/#{relpath()}" + end + + def curr_objdir + "#{objdir_root()}/#{relpath()}" + end + + def srcfile(path) + "#{curr_srcdir()}/#{path}" + end + + def srcexist?(path) + File.exist?(srcfile(path)) + end + + def srcdirectory?(path) + File.dir?(srcfile(path)) + end + + def srcfile?(path) + File.file?(srcfile(path)) + end + + def srcentries(path = '.') + Dir.open("#{curr_srcdir()}/#{path}") {|d| + return d.to_a - %w(. ..) + } + end + + def srcfiles(path = '.') + srcentries(path).select {|fname| + File.file?(File.join(curr_srcdir(), path, fname)) + } + end + + def srcdirectories(path = '.') + srcentries(path).select {|fname| + File.dir?(File.join(curr_srcdir(), path, fname)) + } + end + +end + + +class ToplevelInstaller + + Version = '3.4.1' + Copyright = 'Copyright (c) 2000-2005 Minero Aoki' + + TASKS = [ + [ 'all', 'do config, setup, then install' ], + [ 'config', 'saves your configurations' ], + [ 'show', 'shows current configuration' ], + [ 'setup', 'compiles ruby extentions and others' ], + [ 'install', 'installs files' ], + [ 'test', 'run all tests in test/' ], + [ 'clean', "does `make clean' for each extention" ], + [ 'distclean',"does `make distclean' for each extention" ] + ] + + def ToplevelInstaller.invoke + config = ConfigTable.new(load_rbconfig()) + config.load_standard_entries + config.load_multipackage_entries if multipackage? + config.fixup + klass = (multipackage?() ? ToplevelInstallerMulti : ToplevelInstaller) + klass.new(File.dirname($0), config).invoke + end + + def ToplevelInstaller.multipackage? + File.dir?(File.dirname($0) + '/packages') + end + + def ToplevelInstaller.load_rbconfig + if arg = ARGV.detect {|arg| /\A--rbconfig=/ =~ arg } + ARGV.delete(arg) + load File.expand_path(arg.split(/=/, 2)[1]) + $".push 'rbconfig.rb' + else + require 'rbconfig' + end + ::Config::CONFIG + end + + def initialize(ardir_root, config) + @ardir = File.expand_path(ardir_root) + @config = config + # cache + @valid_task_re = nil + end + + def config(key) + @config[key] + end + + def inspect + "#<#{self.class} #{__id__()}>" + end + + def invoke + run_metaconfigs + case task = parsearg_global() + when nil, 'all' + parsearg_config + init_installers + exec_config + exec_setup + exec_install + else + case task + when 'config', 'test' + ; + when 'clean', 'distclean' + @config.load_savefile if File.exist?(@config.savefile) + else + @config.load_savefile + end + __send__ "parsearg_#{task}" + init_installers + __send__ "exec_#{task}" + end + end + + def run_metaconfigs + @config.load_script "#{@ardir}/metaconfig" + end + + def init_installers + @installer = Installer.new(@config, @ardir, File.expand_path('.')) + end + + # + # Hook Script API bases + # + + def srcdir_root + @ardir + end + + def objdir_root + '.' + end + + def relpath + '.' + end + + # + # Option Parsing + # + + def parsearg_global + while arg = ARGV.shift + case arg + when /\A\w+\z/ + setup_rb_error "invalid task: #{arg}" unless valid_task?(arg) + return arg + when '-q', '--quiet' + @config.verbose = false + when '--verbose' + @config.verbose = true + when '--help' + print_usage $stdout + exit 0 + when '--version' + puts "#{File.basename($0)} version #{Version}" + exit 0 + when '--copyright' + puts Copyright + exit 0 + else + setup_rb_error "unknown global option '#{arg}'" + end + end + nil + end + + def valid_task?(t) + valid_task_re() =~ t + end + + def valid_task_re + @valid_task_re ||= /\A(?:#{TASKS.map {|task,desc| task }.join('|')})\z/ + end + + def parsearg_no_options + unless ARGV.empty? + task = caller(0).first.slice(%r<`parsearg_(\w+)'>, 1) + setup_rb_error "#{task}: unknown options: #{ARGV.join(' ')}" + end + end + + alias parsearg_show parsearg_no_options + alias parsearg_setup parsearg_no_options + alias parsearg_test parsearg_no_options + alias parsearg_clean parsearg_no_options + alias parsearg_distclean parsearg_no_options + + def parsearg_config + evalopt = [] + set = [] + @config.config_opt = [] + while i = ARGV.shift + if /\A--?\z/ =~ i + @config.config_opt = ARGV.dup + break + end + name, value = *@config.parse_opt(i) + if @config.value_config?(name) + @config[name] = value + else + evalopt.push [name, value] + end + set.push name + end + evalopt.each do |name, value| + @config.lookup(name).evaluate value, @config + end + # Check if configuration is valid + set.each do |n| + @config[n] if @config.value_config?(n) + end + end + + def parsearg_install + @config.no_harm = false + @config.install_prefix = '' + while a = ARGV.shift + case a + when '--no-harm' + @config.no_harm = true + when /\A--prefix=/ + path = a.split(/=/, 2)[1] + path = File.expand_path(path) unless path[0,1] == '/' + @config.install_prefix = path + else + setup_rb_error "install: unknown option #{a}" + end + end + end + + def print_usage(out) + out.puts 'Typical Installation Procedure:' + out.puts " $ ruby #{File.basename $0} config" + out.puts " $ ruby #{File.basename $0} setup" + out.puts " # ruby #{File.basename $0} install (may require root privilege)" + out.puts + out.puts 'Detailed Usage:' + out.puts " ruby #{File.basename $0} " + out.puts " ruby #{File.basename $0} [] []" + + fmt = " %-24s %s\n" + out.puts + out.puts 'Global options:' + out.printf fmt, '-q,--quiet', 'suppress message outputs' + out.printf fmt, ' --verbose', 'output messages verbosely' + out.printf fmt, ' --help', 'print this message' + out.printf fmt, ' --version', 'print version and quit' + out.printf fmt, ' --copyright', 'print copyright and quit' + out.puts + out.puts 'Tasks:' + TASKS.each do |name, desc| + out.printf fmt, name, desc + end + + fmt = " %-24s %s [%s]\n" + out.puts + out.puts 'Options for CONFIG or ALL:' + @config.each do |item| + out.printf fmt, item.help_opt, item.description, item.help_default + end + out.printf fmt, '--rbconfig=path', 'rbconfig.rb to load',"running ruby's" + out.puts + out.puts 'Options for INSTALL:' + out.printf fmt, '--no-harm', 'only display what to do if given', 'off' + out.printf fmt, '--prefix=path', 'install path prefix', '' + out.puts + end + + # + # Task Handlers + # + + def exec_config + @installer.exec_config + @config.save # must be final + end + + def exec_setup + @installer.exec_setup + end + + def exec_install + @installer.exec_install + end + + def exec_test + @installer.exec_test + end + + def exec_show + @config.each do |i| + printf "%-20s %s\n", i.name, i.value if i.value? + end + end + + def exec_clean + @installer.exec_clean + end + + def exec_distclean + @installer.exec_distclean + end + +end # class ToplevelInstaller + + +class ToplevelInstallerMulti < ToplevelInstaller + + include FileOperations + + def initialize(ardir_root, config) + super + @packages = directories_of("#{@ardir}/packages") + raise 'no package exists' if @packages.empty? + @root_installer = Installer.new(@config, @ardir, File.expand_path('.')) + end + + def run_metaconfigs + @config.load_script "#{@ardir}/metaconfig", self + @packages.each do |name| + @config.load_script "#{@ardir}/packages/#{name}/metaconfig" + end + end + + attr_reader :packages + + def packages=(list) + raise 'package list is empty' if list.empty? + list.each do |name| + raise "directory packages/#{name} does not exist"\ + unless File.dir?("#{@ardir}/packages/#{name}") + end + @packages = list + end + + def init_installers + @installers = {} + @packages.each do |pack| + @installers[pack] = Installer.new(@config, + "#{@ardir}/packages/#{pack}", + "packages/#{pack}") + end + with = extract_selection(config('with')) + without = extract_selection(config('without')) + @selected = @installers.keys.select {|name| + (with.empty? or with.include?(name)) \ + and not without.include?(name) + } + end + + def extract_selection(list) + a = list.split(/,/) + a.each do |name| + setup_rb_error "no such package: #{name}" unless @installers.key?(name) + end + a + end + + def print_usage(f) + super + f.puts 'Inluded packages:' + f.puts ' ' + @packages.sort.join(' ') + f.puts + end + + # + # Task Handlers + # + + def exec_config + run_hook 'pre-config' + each_selected_installers {|inst| inst.exec_config } + run_hook 'post-config' + @config.save # must be final + end + + def exec_setup + run_hook 'pre-setup' + each_selected_installers {|inst| inst.exec_setup } + run_hook 'post-setup' + end + + def exec_install + run_hook 'pre-install' + each_selected_installers {|inst| inst.exec_install } + run_hook 'post-install' + end + + def exec_test + run_hook 'pre-test' + each_selected_installers {|inst| inst.exec_test } + run_hook 'post-test' + end + + def exec_clean + rm_f @config.savefile + run_hook 'pre-clean' + each_selected_installers {|inst| inst.exec_clean } + run_hook 'post-clean' + end + + def exec_distclean + rm_f @config.savefile + run_hook 'pre-distclean' + each_selected_installers {|inst| inst.exec_distclean } + run_hook 'post-distclean' + end + + # + # lib + # + + def each_selected_installers + Dir.mkdir 'packages' unless File.dir?('packages') + @selected.each do |pack| + $stderr.puts "Processing the package `#{pack}' ..." if verbose? + Dir.mkdir "packages/#{pack}" unless File.dir?("packages/#{pack}") + Dir.chdir "packages/#{pack}" + yield @installers[pack] + Dir.chdir '../..' + end + end + + def run_hook(id) + @root_installer.run_hook id + end + + # module FileOperations requires this + def verbose? + @config.verbose? + end + + # module FileOperations requires this + def no_harm? + @config.no_harm? + end + +end # class ToplevelInstallerMulti + + +class Installer + + FILETYPES = %w( bin lib ext data conf man ) + + include FileOperations + include HookScriptAPI + + def initialize(config, srcroot, objroot) + @config = config + @srcdir = File.expand_path(srcroot) + @objdir = File.expand_path(objroot) + @currdir = '.' + end + + def inspect + "#<#{self.class} #{File.basename(@srcdir)}>" + end + + def noop(rel) + end + + # + # Hook Script API base methods + # + + def srcdir_root + @srcdir + end + + def objdir_root + @objdir + end + + def relpath + @currdir + end + + # + # Config Access + # + + # module FileOperations requires this + def verbose? + @config.verbose? + end + + # module FileOperations requires this + def no_harm? + @config.no_harm? + end + + def verbose_off + begin + save, @config.verbose = @config.verbose?, false + yield + ensure + @config.verbose = save + end + end + + # + # TASK config + # + + def exec_config + exec_task_traverse 'config' + end + + alias config_dir_bin noop + alias config_dir_lib noop + + def config_dir_ext(rel) + extconf if extdir?(curr_srcdir()) + end + + alias config_dir_data noop + alias config_dir_conf noop + alias config_dir_man noop + + def extconf + ruby "#{curr_srcdir()}/extconf.rb", *@config.config_opt + end + + # + # TASK setup + # + + def exec_setup + exec_task_traverse 'setup' + end + + def setup_dir_bin(rel) + files_of(curr_srcdir()).each do |fname| + update_shebang_line "#{curr_srcdir()}/#{fname}" + end + end + + alias setup_dir_lib noop + + def setup_dir_ext(rel) + make if extdir?(curr_srcdir()) + end + + alias setup_dir_data noop + alias setup_dir_conf noop + alias setup_dir_man noop + + def update_shebang_line(path) + return if no_harm? + return if config('shebang') == 'never' + old = Shebang.load(path) + if old + $stderr.puts "warning: #{path}: Shebang line includes too many args. It is not portable and your program may not work." if old.args.size > 1 + new = new_shebang(old) + return if new.to_s == old.to_s + else + return unless config('shebang') == 'all' + new = Shebang.new(config('rubypath')) + end + $stderr.puts "updating shebang: #{File.basename(path)}" if verbose? + open_atomic_writer(path) {|output| + File.open(path, 'rb') {|f| + f.gets if old # discard + output.puts new.to_s + output.print f.read + } + } + end + + def new_shebang(old) + if /\Aruby/ =~ File.basename(old.cmd) + Shebang.new(config('rubypath'), old.args) + elsif File.basename(old.cmd) == 'env' and old.args.first == 'ruby' + Shebang.new(config('rubypath'), old.args[1..-1]) + else + return old unless config('shebang') == 'all' + Shebang.new(config('rubypath')) + end + end + + def open_atomic_writer(path, &block) + tmpfile = File.basename(path) + '.tmp' + begin + File.open(tmpfile, 'wb', &block) + File.rename tmpfile, File.basename(path) + ensure + File.unlink tmpfile if File.exist?(tmpfile) + end + end + + class Shebang + def Shebang.load(path) + line = nil + File.open(path) {|f| + line = f.gets + } + return nil unless /\A#!/ =~ line + parse(line) + end + + def Shebang.parse(line) + cmd, *args = *line.strip.sub(/\A\#!/, '').split(' ') + new(cmd, args) + end + + def initialize(cmd, args = []) + @cmd = cmd + @args = args + end + + attr_reader :cmd + attr_reader :args + + def to_s + "#! #{@cmd}" + (@args.empty? ? '' : " #{@args.join(' ')}") + end + end + + # + # TASK install + # + + def exec_install + rm_f 'InstalledFiles' + exec_task_traverse 'install' + end + + def install_dir_bin(rel) + install_files targetfiles(), "#{config('bindir')}/#{rel}", 0755 + end + + def install_dir_lib(rel) + install_files libfiles(), "#{config('rbdir')}/#{rel}", 0644 + end + + def install_dir_ext(rel) + return unless extdir?(curr_srcdir()) + install_files rubyextentions('.'), + "#{config('sodir')}/#{File.dirname(rel)}", + 0555 + end + + def install_dir_data(rel) + install_files targetfiles(), "#{config('datadir')}/#{rel}", 0644 + end + + def install_dir_conf(rel) + # FIXME: should not remove current config files + # (rename previous file to .old/.org) + install_files targetfiles(), "#{config('sysconfdir')}/#{rel}", 0644 + end + + def install_dir_man(rel) + install_files targetfiles(), "#{config('mandir')}/#{rel}", 0644 + end + + def install_files(list, dest, mode) + mkdir_p dest, @config.install_prefix + list.each do |fname| + install fname, dest, mode, @config.install_prefix + end + end + + def libfiles + glob_reject(%w(*.y *.output), targetfiles()) + end + + def rubyextentions(dir) + ents = glob_select("*.#{@config.dllext}", targetfiles()) + if ents.empty? + setup_rb_error "no ruby extention exists: 'ruby #{$0} setup' first" + end + ents + end + + def targetfiles + mapdir(existfiles() - hookfiles()) + end + + def mapdir(ents) + ents.map {|ent| + if File.exist?(ent) + then ent # objdir + else "#{curr_srcdir()}/#{ent}" # srcdir + end + } + end + + # picked up many entries from cvs-1.11.1/src/ignore.c + JUNK_FILES = %w( + core RCSLOG tags TAGS .make.state + .nse_depinfo #* .#* cvslog.* ,* .del-* *.olb + *~ *.old *.bak *.BAK *.orig *.rej _$* *$ + + *.org *.in .* + ) + + def existfiles + glob_reject(JUNK_FILES, (files_of(curr_srcdir()) | files_of('.'))) + end + + def hookfiles + %w( pre-%s post-%s pre-%s.rb post-%s.rb ).map {|fmt| + %w( config setup install clean ).map {|t| sprintf(fmt, t) } + }.flatten + end + + def glob_select(pat, ents) + re = globs2re([pat]) + ents.select {|ent| re =~ ent } + end + + def glob_reject(pats, ents) + re = globs2re(pats) + ents.reject {|ent| re =~ ent } + end + + GLOB2REGEX = { + '.' => '\.', + '$' => '\$', + '#' => '\#', + '*' => '.*' + } + + def globs2re(pats) + /\A(?:#{ + pats.map {|pat| pat.gsub(/[\.\$\#\*]/) {|ch| GLOB2REGEX[ch] } }.join('|') + })\z/ + end + + # + # TASK test + # + + TESTDIR = 'test' + + def exec_test + unless File.directory?('test') + $stderr.puts 'no test in this package' if verbose? + return + end + $stderr.puts 'Running tests...' if verbose? + begin + require 'test/unit' + rescue LoadError + setup_rb_error 'test/unit cannot loaded. You need Ruby 1.8 or later to invoke this task.' + end + runner = Test::Unit::AutoRunner.new(true) + runner.to_run << TESTDIR + runner.run + end + + # + # TASK clean + # + + def exec_clean + exec_task_traverse 'clean' + rm_f @config.savefile + rm_f 'InstalledFiles' + end + + alias clean_dir_bin noop + alias clean_dir_lib noop + alias clean_dir_data noop + alias clean_dir_conf noop + alias clean_dir_man noop + + def clean_dir_ext(rel) + return unless extdir?(curr_srcdir()) + make 'clean' if File.file?('Makefile') + end + + # + # TASK distclean + # + + def exec_distclean + exec_task_traverse 'distclean' + rm_f @config.savefile + rm_f 'InstalledFiles' + end + + alias distclean_dir_bin noop + alias distclean_dir_lib noop + + def distclean_dir_ext(rel) + return unless extdir?(curr_srcdir()) + make 'distclean' if File.file?('Makefile') + end + + alias distclean_dir_data noop + alias distclean_dir_conf noop + alias distclean_dir_man noop + + # + # Traversing + # + + def exec_task_traverse(task) + run_hook "pre-#{task}" + FILETYPES.each do |type| + if type == 'ext' and config('without-ext') == 'yes' + $stderr.puts 'skipping ext/* by user option' if verbose? + next + end + traverse task, type, "#{task}_dir_#{type}" + end + run_hook "post-#{task}" + end + + def traverse(task, rel, mid) + dive_into(rel) { + run_hook "pre-#{task}" + __send__ mid, rel.sub(%r[\A.*?(?:/|\z)], '') + directories_of(curr_srcdir()).each do |d| + traverse task, "#{rel}/#{d}", mid + end + run_hook "post-#{task}" + } + end + + def dive_into(rel) + return unless File.dir?("#{@srcdir}/#{rel}") + + dir = File.basename(rel) + Dir.mkdir dir unless File.dir?(dir) + prevdir = Dir.pwd + Dir.chdir dir + $stderr.puts '---> ' + rel if verbose? + @currdir = rel + yield + Dir.chdir prevdir + $stderr.puts '<--- ' + rel if verbose? + @currdir = File.dirname(rel) + end + + def run_hook(id) + path = [ "#{curr_srcdir()}/#{id}", + "#{curr_srcdir()}/#{id}.rb" ].detect {|cand| File.file?(cand) } + return unless path + begin + instance_eval File.read(path), path, 1 + rescue + raise if $DEBUG + setup_rb_error "hook #{path} failed:\n" + $!.message + end + end + +end # class Installer + + +class SetupError < StandardError; end + +def setup_rb_error(msg) + raise SetupError, msg +end + +if $0 == __FILE__ + begin + ToplevelInstaller.invoke + rescue SetupError + raise if $DEBUG + $stderr.puts $!.message + $stderr.puts "Try 'ruby #{$0} --help' for detailed usage." + exit 1 + end +end diff --git a/ruby/gem/tasks/deployment.rake b/ruby/gem/tasks/deployment.rake new file mode 100644 index 0000000..2f43742 --- /dev/null +++ b/ruby/gem/tasks/deployment.rake @@ -0,0 +1,34 @@ +desc 'Release the website and new gem version' +task :deploy => [:check_version, :website, :release] do + puts "Remember to create SVN tag:" + puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " + + "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} " + puts "Suggested comment:" + puts "Tagging release #{CHANGES}" +end + +desc 'Runs tasks website_generate and install_gem as a local deployment of the gem' +task :local_deploy => [:website_generate, :install_gem] + +task :check_version do + unless ENV['VERSION'] + puts 'Must pass a VERSION=x.y.z release version' + exit + end + unless ENV['VERSION'] == VERS + puts "Please update your version.rb to match the release version, currently #{VERS}" + exit + end +end + +desc 'Install the package as a gem, without generating documentation(ri/rdoc)' +task :install_gem_no_doc => [:clean, :package] do + sh "#{'sudo ' unless Hoe::WINDOZE }gem install pkg/*.gem --no-rdoc --no-ri" +end + +namespace :manifest do + desc 'Recreate Manifest.txt to include ALL files' + task :refresh do + `rake check_manifest | patch -p0 > Manifest.txt` + end +end \ No newline at end of file diff --git a/ruby/gem/tasks/environment.rake b/ruby/gem/tasks/environment.rake new file mode 100644 index 0000000..691ed3b --- /dev/null +++ b/ruby/gem/tasks/environment.rake @@ -0,0 +1,7 @@ +task :ruby_env do + RUBY_APP = if RUBY_PLATFORM =~ /java/ + "jruby" + else + "ruby" + end unless defined? RUBY_APP +end diff --git a/ruby/pack.c b/ruby/pack.c new file mode 100644 index 0000000..3520f9f --- /dev/null +++ b/ruby/pack.c @@ -0,0 +1,131 @@ +/* + * MessagePack packing routine for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pack_inline.h" + +#ifndef RUBY_VM +#include "st.h" // ruby hash +#endif + +static ID s_to_msgpack; + +#define ARG_BUFFER(name, argc, argv) \ + VALUE name; \ + if(argc == 1) { \ + name = argv[0]; \ + } else if(argc == 0) { \ + name = rb_str_buf_new(0); \ + } else { \ + rb_raise(rb_eArgError, "wrong number of arguments (%d for 0)", argc); \ + } + +static VALUE MessagePack_NilClass_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_nil(out); + return out; +} + +static VALUE MessagePack_TrueClass_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_true(out); + return out; +} + +static VALUE MessagePack_FalseClass_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_false(out); + return out; +} + + +static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_int(out, FIX2INT(self)); + return out; +} + +static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_double(out, rb_num2dbl(self)); + return out; +} + +static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_raw(out, RSTRING_PTR(self), RSTRING_LEN(self)); + return out; +} + +static VALUE MessagePack_Array_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_array(out, RARRAY_LEN(self)); + VALUE* p = RARRAY_PTR(self); + VALUE* const pend = p + RARRAY_LEN(self); + for(;p != pend; ++p) { + rb_funcall(*p, s_to_msgpack, 1, out); + } + return out; +} + +#ifndef RHASH_SIZE // Ruby 1.8 +#define RHASH_SIZE(h) (RHASH(h)->tbl ? RHASH(h)->tbl->num_entries : 0) +#endif + +static int MessagePack_Hash_to_msgpack_foreach(VALUE key, VALUE value, VALUE out) +{ + if (key == Qundef) { return ST_CONTINUE; } + rb_funcall(key, s_to_msgpack, 1, out); + rb_funcall(value, s_to_msgpack, 1, out); + return ST_CONTINUE; +} + +static VALUE MessagePack_Hash_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_map(out, RHASH_SIZE(self)); + rb_hash_foreach(self, MessagePack_Hash_to_msgpack_foreach, out); + return out; +} + + +static VALUE MessagePack_pack(VALUE self, VALUE data) +{ + return rb_funcall(data, s_to_msgpack, 0); +} + + +void Init_msgpack_pack(VALUE mMessagePack) +{ + s_to_msgpack = rb_intern("to_msgpack"); + rb_define_method_id(rb_cNilClass, s_to_msgpack, MessagePack_NilClass_to_msgpack, -1); + rb_define_method_id(rb_cTrueClass, s_to_msgpack, MessagePack_TrueClass_to_msgpack, -1); + rb_define_method_id(rb_cFalseClass, s_to_msgpack, MessagePack_FalseClass_to_msgpack, -1); + rb_define_method_id(rb_cFixnum, s_to_msgpack, MessagePack_Fixnum_to_msgpack, -1); + rb_define_method_id(rb_cFloat, s_to_msgpack, MessagePack_Float_to_msgpack, -1); + rb_define_method_id(rb_cString, s_to_msgpack, MessagePack_String_to_msgpack, -1); + rb_define_method_id(rb_cArray, s_to_msgpack, MessagePack_Array_to_msgpack, -1); + rb_define_method_id(rb_cHash, s_to_msgpack, MessagePack_Hash_to_msgpack, -1); + rb_define_module_function(mMessagePack, "pack", MessagePack_pack, 1); +} + diff --git a/ruby/pack.h b/ruby/pack.h new file mode 100644 index 0000000..c38ac48 --- /dev/null +++ b/ruby/pack.h @@ -0,0 +1,26 @@ +/* + * MessagePack packing routine for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef PACK_H__ +#define PACK_H__ + +#include "ruby.h" + +void Init_msgpack_pack(VALUE mMessagePack); + +#endif /* pack.h */ + diff --git a/ruby/pack_inline.h b/ruby/pack_inline.h new file mode 100644 index 0000000..ab4b092 --- /dev/null +++ b/ruby/pack_inline.h @@ -0,0 +1,33 @@ +/* + * MessagePack packing routine for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef PACK_INLINE_H__ +#define PACK_INLINE_H__ + +#include "ruby.h" + +typedef VALUE msgpack_pack_context; + +static inline void msgpack_pack_append_buffer(VALUE x, const unsigned char* b, unsigned int l) +{ + rb_str_buf_cat(x, (const void*)b, l); +} + +#include "msgpack/pack/inline_impl.h" + +#endif /* pack_inline.h */ + diff --git a/ruby/rbinit.c b/ruby/rbinit.c new file mode 100644 index 0000000..7ef92fb --- /dev/null +++ b/ruby/rbinit.c @@ -0,0 +1,29 @@ +/* + * MessagePack for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pack.h" +#include "unpack.h" + +static VALUE mMessagePack; + +void Init_msgpack(void) +{ + mMessagePack = rb_define_module("MessagePack"); + Init_msgpack_unpack(mMessagePack); + Init_msgpack_pack(mMessagePack); +} + diff --git a/ruby/test_format.rb b/ruby/test_format.rb new file mode 100644 index 0000000..7c2e8fc --- /dev/null +++ b/ruby/test_format.rb @@ -0,0 +1,128 @@ +require 'msgpack' + +@up = MessagePack::Unpacker.new + +def check(bytes, should) + puts "----" + @up.reset + src = bytes.pack('C*') + ret = @up.execute(src, 0) + if ret != src.length + puts "** EXTRA BYTES **" + end + puts bytes.map{|x|"%x"%x}.join(' ') + data = @up.data + p data + if data != should + puts "** TEST FAILED **" + p should + end +end + +# SimpleValue +check([ + 0x93, 0xc0, 0xc2, 0xc3, +], [nil,false,true]) + +# Fixnum +check([ + 0x92, + 0x93, 0x00, 0x40, 0x7f, + 0x93, 0xe0, 0xf0, 0xff, +], [[0,64,127], [-32,-16,-1]]) + +# FixArray +check([ + 0x92, + 0x90, + 0x91, + 0x91, 0xc0, +], [[],[[nil]]]) + + +# FixRaw +check([ + 0x94, + 0xa0, + 0xa1, ?a, + 0xa2, ?b, ?c, + 0xa3, ?d, ?e, ?f, +], ["","a","bc","def"]) + +# FixMap +check([ + 0x82, + 0xc2, 0x81, + 0xc0, 0xc0, + 0xc3, 0x81, + 0xc0, 0x80, +], {false=>{nil=>nil}, true=>{nil=>{}}}) + +# unsigned int +check([ + 0x99, + 0xcc, 0, + 0xcc, 128, + 0xcc, 255, + 0xcd, 0x00, 0x00, + 0xcd, 0x80, 0x00, + 0xcd, 0xff, 0xff, + 0xce, 0x00, 0x00, 0x00, 0x00, + 0xce, 0x80, 0x00, 0x00, 0x00, + 0xce, 0xff, 0xff, 0xff, 0xff, +], [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295]) + +# signed int +check([ + 0x99, + 0xd0, 0, + 0xd0, 128, + 0xd0, 255, + 0xd1, 0x00, 0x00, + 0xd1, 0x80, 0x00, + 0xd1, 0xff, 0xff, + 0xd2, 0x00, 0x00, 0x00, 0x00, + 0xd2, 0x80, 0x00, 0x00, 0x00, + 0xd2, 0xff, 0xff, 0xff, 0xff, +], [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1]) + +# raw +check([ + 0x96, + 0xda, 0x00, 0x00, + 0xda, 0x00, 0x01, ?a, + 0xda, 0x00, 0x02, ?a, ?b, + 0xdb, 0x00, 0x00, 0x00, 0x00, + 0xdb, 0x00, 0x00, 0x00, 0x01, ?a, + 0xdb, 0x00, 0x00, 0x00, 0x02, ?a, ?b, +], ["", "a", "ab", "", "a", "ab"]) + +# array +check([ + 0x96, + 0xdc, 0x00, 0x00, + 0xdc, 0x00, 0x01, 0xc0, + 0xdc, 0x00, 0x02, 0xc2, 0xc3, + 0xdd, 0x00, 0x00, 0x00, 0x00, + 0xdd, 0x00, 0x00, 0x00, 0x01, 0xc0, + 0xdd, 0x00, 0x00, 0x00, 0x02, 0xc2, 0xc3 +], [[], [nil], [false,true], [], [nil], [false,true]]) + +# map +check([ + 0x96, + 0xde, 0x00, 0x00, + 0xde, 0x00, 0x01, 0xc0, 0xc2, + 0xde, 0x00, 0x02, 0xc0, 0xc2, 0xc3, 0xc2, + 0xdf, 0x00, 0x00, 0x00, 0x00, + 0xdf, 0x00, 0x00, 0x00, 0x01, 0xc0, 0xc2, + 0xdf, 0x00, 0x00, 0x00, 0x02, 0xc0, 0xc2, 0xc3, 0xc2, +], [{}, {nil=>false}, {true=>false, nil=>false}, {}, {nil=>false}, {true=>false, nil=>false}]) + +# string +check([ + 0x92, + 0xc1, 0x00, + 0xc1, ?a, ?b, ?c, 0x00, +], ["", "abc"]) + diff --git a/ruby/test_pack.rb b/ruby/test_pack.rb new file mode 100644 index 0000000..16a8ccf --- /dev/null +++ b/ruby/test_pack.rb @@ -0,0 +1,56 @@ +require 'msgpack' + +def check(data) + puts "---" + pack = data.to_msgpack + p data + puts pack.unpack('C*').map{|x|"%02x"%x}.join(' ') + re = MessagePack::unpack(pack) + if re != data + p re + puts "** TEST FAILED **" + end +end + +check 0 +check 1 +check 127 +check 128 +check 255 +check 256 +check 65535 +check 65536 +check -1 +check -128 +check -129 +check -32768 +check -32769 + +check 1.0 + +check "" +check "a" +check "a"*31 +check "a"*32 + +check nil +check true +check false + +check [] +check [[]] +check [[], nil] + +check( {nil=>0} ) + +check (1<<23) +__END__ + +ary = [] +i = 0 +while i < (1<<16) + ary << i + i += 1 +end +check ary + diff --git a/ruby/unpack.c b/ruby/unpack.c new file mode 100644 index 0000000..fa2996d --- /dev/null +++ b/ruby/unpack.c @@ -0,0 +1,202 @@ +/* + * MessagePack unpacking routine for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "ruby.h" +#include "unpack_context.h" +#include + +#define UNPACKER(from, name) \ + msgpack_unpacker *name = NULL; \ + Data_Get_Struct(from, msgpack_unpacker, name); \ + if(name == NULL) { \ + rb_raise(rb_eArgError, "NULL found for " # name " when shouldn't be."); \ + } + +#define CHECK_STRING_TYPE(value) \ + value = rb_check_string_type(value); \ + if( NIL_P(value) ) { \ + rb_raise(rb_eTypeError, "instance of String needed"); \ + } + +static VALUE cUnpacker; +static VALUE eUnpackError; + +static void MessagePack_Unpacker_free(void* data) +{ + if(data) { free(data); } +} + +static void MessagePack_Unpacker_mark(msgpack_unpacker *mp) +{ + unsigned int i; + for(i=0; i < mp->top; ++i) { + rb_gc_mark(mp->stack[i].obj); + rb_gc_mark(mp->stack[i].tmp.map_key); + } +} + +static VALUE MessagePack_Unpacker_alloc(VALUE klass) +{ + VALUE obj; + msgpack_unpacker* mp = ALLOC_N(msgpack_unpacker, 1); + obj = Data_Wrap_Struct(klass, MessagePack_Unpacker_mark, + MessagePack_Unpacker_free, mp); + return obj; +} + +static VALUE MessagePack_Unpacker_reset(VALUE self) +{ + UNPACKER(self, mp); + mp->user.finished = false; + msgpack_unpacker_init(mp); + return self; +} + +static VALUE MessagePack_Unpacker_initialize(VALUE self) +{ + return MessagePack_Unpacker_reset(self); +} + + +static VALUE MessagePack_Unpacker_execute_impl(VALUE args) +{ + VALUE self = ((VALUE*)args)[0]; + VALUE data = ((VALUE*)args)[1]; + VALUE off = ((VALUE*)args)[2]; + + UNPACKER(self, mp); + size_t from = NUM2UINT(off); + char* dptr = RSTRING_PTR(data); + long dlen = RSTRING_LEN(data); + int ret; + + if(from >= dlen) { + rb_raise(eUnpackError, "Requested start is after data buffer end."); + } + + ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); + + if(ret < 0) { + rb_raise(eUnpackError, "Parse error."); + } else if(ret > 0) { + mp->user.finished = true; + return ULONG2NUM(from); + } else { + mp->user.finished = false; + return ULONG2NUM(from); + } +} + +static VALUE MessagePack_Unpacker_execute_rescue(VALUE nouse) +{ + rb_gc_enable(); +#ifdef RUBY_VM + rb_exc_raise(rb_errinfo()); +#else + rb_exc_raise(ruby_errinfo); +#endif +} + +static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) +{ + // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ + rb_gc_disable(); + VALUE args[3] = {self, data, off}; + VALUE ret = rb_rescue(MessagePack_Unpacker_execute_impl, (VALUE)args, + MessagePack_Unpacker_execute_rescue, Qnil); + rb_gc_enable(); + return ret; +} + +static VALUE MessagePack_Unpacker_finished_p(VALUE self) +{ + UNPACKER(self, mp); + if(mp->user.finished) { + return Qtrue; + } + return Qfalse; +} + +static VALUE MessagePack_Unpacker_data(VALUE self) +{ + UNPACKER(self, mp); + return msgpack_unpacker_data(mp); +} + + +static VALUE MessagePack_unpack_impl(VALUE args) +{ + msgpack_unpacker* mp = (msgpack_unpacker*)((VALUE*)args)[0]; + VALUE data = ((VALUE*)args)[1]; + + size_t from = 0; + char* dptr = RSTRING_PTR(data); + long dlen = RSTRING_LEN(data); + int ret; + + ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); + + if(ret < 0) { + rb_raise(eUnpackError, "Parse error."); + } else if(ret == 0) { + rb_raise(eUnpackError, "Insufficient bytes."); + } else { + if(from < dlen) { + rb_raise(eUnpackError, "Extra bytes."); + } + return msgpack_unpacker_data(mp); + } +} + +static VALUE MessagePack_unpack_rescue(VALUE args) +{ + rb_gc_enable(); +#ifdef RUBY_VM + rb_exc_raise(rb_errinfo()); +#else + rb_exc_raise(ruby_errinfo); +#endif +} + +static VALUE MessagePack_unpack(VALUE self, VALUE data) +{ + CHECK_STRING_TYPE(data); + msgpack_unpacker mp; + msgpack_unpacker_init(&mp); + rb_gc_disable(); + VALUE args[2] = {(VALUE)&mp, data}; + VALUE ret = rb_rescue(MessagePack_unpack_impl, (VALUE)args, + MessagePack_unpack_rescue, Qnil); + rb_gc_enable(); + return ret; +} + + +void Init_msgpack_unpack(VALUE mMessagePack) +{ + eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); + cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); + rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); + rb_define_method(cUnpacker, "initialize", MessagePack_Unpacker_initialize, 0); + rb_define_method(cUnpacker, "execute", MessagePack_Unpacker_execute, 2); + rb_define_method(cUnpacker, "finished?", MessagePack_Unpacker_finished_p, 0); + rb_define_method(cUnpacker, "data", MessagePack_Unpacker_data, 0); + rb_define_method(cUnpacker, "reset", MessagePack_Unpacker_reset, 0); + rb_define_module_function(mMessagePack, "unpack", MessagePack_unpack, 1); +} + + diff --git a/ruby/unpack.h b/ruby/unpack.h new file mode 100644 index 0000000..0fe01ec --- /dev/null +++ b/ruby/unpack.h @@ -0,0 +1,26 @@ +/* + * MessagePack unpacking routine for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef UNPACK_H__ +#define UNPACK_H__ + +#include "ruby.h" + +void Init_msgpack_unpack(VALUE mMessagePack); + +#endif /* unpack.h */ + diff --git a/ruby/unpack_context.h b/ruby/unpack_context.h new file mode 100644 index 0000000..35e0132 --- /dev/null +++ b/ruby/unpack_context.h @@ -0,0 +1,35 @@ +/* + * MessagePack unpacking routine for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef UNPACK_CONTEXT_H__ +#define UNPACK_CONTEXT_H__ + +#include "ruby.h" +#include +#include + +typedef VALUE msgpack_object; + +typedef struct { + bool finished; +} msgpack_unpack_context; + + +#include "msgpack/unpack/inline_context.h" + +#endif /* unpack_context.h */ + diff --git a/ruby/unpack_inline.c b/ruby/unpack_inline.c new file mode 100644 index 0000000..fa684c9 --- /dev/null +++ b/ruby/unpack_inline.c @@ -0,0 +1,81 @@ +/* + * MessagePack unpacking routine for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "unpack_context.h" + +static inline VALUE msgpack_unpack_init(msgpack_unpack_context* x) +{ return Qnil; } + +static inline VALUE msgpack_unpack_unsigned_int_8(msgpack_unpack_context* x, uint8_t d) +{ return INT2FIX(d); } + +static inline VALUE msgpack_unpack_unsigned_int_16(msgpack_unpack_context* x, uint16_t d) +{ return INT2FIX(d); } + +static inline VALUE msgpack_unpack_unsigned_int_32(msgpack_unpack_context* x, uint32_t d) +{ return UINT2NUM(d); } + +static inline VALUE msgpack_unpack_unsigned_int_64(msgpack_unpack_context* x, uint64_t d) +{ return UINT2NUM(d); } // FIXME + +static inline VALUE msgpack_unpack_signed_int_8(msgpack_unpack_context* x, int8_t d) +{ return INT2FIX((long)d); } + +static inline VALUE msgpack_unpack_signed_int_16(msgpack_unpack_context* x, int16_t d) +{ return INT2FIX((long)d); } + +static inline VALUE msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int32_t d) +{ return INT2NUM((long)d); } + +static inline VALUE msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d) +{ return INT2NUM(d); } // FIXME + +static inline VALUE msgpack_unpack_float(msgpack_unpack_context* x, float d) +{ return rb_float_new(d); } + +static inline VALUE msgpack_unpack_double(msgpack_unpack_context* x, double d) +{ return rb_float_new(d); } + +static inline VALUE msgpack_unpack_nil(msgpack_unpack_context* x) +{ return Qnil; } + +static inline VALUE msgpack_unpack_true(msgpack_unpack_context* x) +{ return Qtrue; } + +static inline VALUE msgpack_unpack_false(msgpack_unpack_context* x) +{ return Qfalse; } + +static inline VALUE msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned int n) +{ return rb_ary_new2(n); } + +static inline void msgpack_unpack_array_item(msgpack_unpack_context* x, VALUE c, VALUE o) +{ rb_ary_push(c, o); } + +static inline VALUE msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n) +{ return rb_hash_new(); } + +static inline void msgpack_unpack_map_item(msgpack_unpack_context* x, VALUE c, VALUE k, VALUE v) +{ rb_hash_aset(c, k, v); } + +static inline VALUE msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l) +{ return rb_str_new(b, l); } + +static inline VALUE msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, size_t l) +{ return rb_str_new(b, l); } + +#include "msgpack/unpack/inline_impl.h" + From 9f460f17d73e749d58c2f58beb621e1f7292cfa6 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:55 +0000 Subject: [PATCH 0002/1172] lang/c/msgpack: uint64_t, int64_t support for ruby git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@49 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- ruby/Makefile | 153 ------------------------------------------- ruby/pack.c | 18 +++++ ruby/unpack_inline.c | 6 +- 3 files changed, 21 insertions(+), 156 deletions(-) delete mode 100644 ruby/Makefile diff --git a/ruby/Makefile b/ruby/Makefile deleted file mode 100644 index 6ef04b0..0000000 --- a/ruby/Makefile +++ /dev/null @@ -1,153 +0,0 @@ - -SHELL = /bin/sh - -#### Start of system configuration section. #### - -srcdir = . -topdir = /Users/frsyuki/ports/lib/ruby/1.8/i686-darwin9.1.0 -hdrdir = $(topdir) -VPATH = $(srcdir):$(topdir):$(hdrdir) -prefix = $(DESTDIR)/Users/frsyuki/ports -exec_prefix = $(prefix) -sitedir = $(prefix)/lib/ruby/site_ruby -rubylibdir = $(libdir)/ruby/$(ruby_version) -docdir = $(datarootdir)/doc/$(PACKAGE) -dvidir = $(docdir) -datarootdir = $(prefix)/share -archdir = $(rubylibdir)/$(arch) -sbindir = $(exec_prefix)/sbin -psdir = $(docdir) -vendordir = $(prefix)/lib/ruby/vendor_ruby -localedir = $(datarootdir)/locale -htmldir = $(docdir) -datadir = $(datarootdir) -includedir = $(prefix)/include -infodir = $(datarootdir)/info -sysconfdir = $(prefix)/etc -mandir = $(DESTDIR)/Users/frsyuki/ports/share/man -libdir = $(exec_prefix)/lib -sharedstatedir = $(prefix)/com -oldincludedir = $(DESTDIR)/usr/include -pdfdir = $(docdir) -sitearchdir = $(sitelibdir)/$(sitearch) -vendorarchdir = $(vendorlibdir)/$(vendorarch) -bindir = $(exec_prefix)/bin -localstatedir = $(prefix)/var -vendorlibdir = $(vendordir)/$(ruby_version) -sitelibdir = $(sitedir)/$(ruby_version) -libexecdir = $(exec_prefix)/libexec - -CC = /usr/bin/gcc-4.0 -LIBRUBY = $(LIBRUBY_SO) -LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a -LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME) -LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static - -RUBY_EXTCONF_H = -CFLAGS = -fno-common -O2 -fno-common -pipe -fno-common -I.. -Wall -O9 -INCFLAGS = -I. -I$(topdir) -I$(hdrdir) -I$(srcdir) -CPPFLAGS = -I/Users/frsyuki/ports/include -CXXFLAGS = $(CFLAGS) -DLDFLAGS = -L. -L/Users/frsyuki/ports/lib -LDSHARED = cc -dynamic -bundle -undefined suppress -flat_namespace -AR = ar -EXEEXT = - -RUBY_INSTALL_NAME = ruby -RUBY_SO_NAME = ruby -arch = i686-darwin9.1.0 -sitearch = i686-darwin9.1.0 -vendorarch = i686-darwin9.1.0 -ruby_version = 1.8 -ruby = /Users/frsyuki/ports/bin/ruby -RUBY = $(ruby) -RM = rm -f -MAKEDIRS = mkdir -p -INSTALL = /usr/bin/install -INSTALL_PROG = $(INSTALL) -m 0755 -INSTALL_DATA = $(INSTALL) -m 644 -COPY = cp - -#### End of system configuration section. #### - -preload = - -libpath = . $(libdir) -LIBPATH = -L"." -L"$(libdir)" -DEFFILE = - -CLEANFILES = mkmf.log -DISTCLEANFILES = - -extout = -extout_prefix = -target_prefix = -LOCAL_LIBS = -LIBS = $(LIBRUBYARG_SHARED) -lpthread -ldl -lobjc -SRCS = pack.c rbinit.c unpack.c unpack_inline.c -OBJS = pack.o rbinit.o unpack.o unpack_inline.o -TARGET = msgpack -DLLIB = $(TARGET).bundle -EXTSTATIC = -STATIC_LIB = - -RUBYCOMMONDIR = $(sitedir)$(target_prefix) -RUBYLIBDIR = $(sitelibdir)$(target_prefix) -RUBYARCHDIR = $(sitearchdir)$(target_prefix) - -TARGET_SO = $(DLLIB) -CLEANLIBS = $(TARGET).bundle $(TARGET).il? $(TARGET).tds $(TARGET).map -CLEANOBJS = *.o *.a *.s[ol] *.pdb *.exp *.bak - -all: $(DLLIB) -static: $(STATIC_LIB) - -clean: - @-$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) - -distclean: clean - @-$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log - @-$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES) - -realclean: distclean -install: install-so install-rb - -install-so: $(RUBYARCHDIR) -install-so: $(RUBYARCHDIR)/$(DLLIB) -$(RUBYARCHDIR)/$(DLLIB): $(DLLIB) - $(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR) -install-rb: pre-install-rb install-rb-default -install-rb-default: pre-install-rb-default -pre-install-rb: Makefile -pre-install-rb-default: Makefile -$(RUBYARCHDIR): - $(MAKEDIRS) $@ - -site-install: site-install-so site-install-rb -site-install-so: install-so -site-install-rb: install-rb - -.SUFFIXES: .c .m .cc .cxx .cpp .C .o - -.cc.o: - $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< - -.cxx.o: - $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< - -.cpp.o: - $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< - -.C.o: - $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< - -.c.o: - $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) -c $< - -$(DLLIB): $(OBJS) - @-$(RM) $@ - $(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS) - - - -$(OBJS): ruby.h defines.h diff --git a/ruby/pack.c b/ruby/pack.c index 3520f9f..3d71776 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -62,6 +62,23 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + +#ifndef RBIGNUM_SIGN // Ruby 1.8 +#define RBIGNUM_SIGN(b) (RBIGNUM(b)->sign) +#endif + +static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + // FIXME bignum + if(RBIGNUM_SIGN(self)) { // positive + msgpack_pack_unsigned_int_64(out, rb_big2ull(self)); + } else { // negative + msgpack_pack_signed_int_64(out, rb_big2ll(self)); + } + return out; +} + static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -122,6 +139,7 @@ void Init_msgpack_pack(VALUE mMessagePack) rb_define_method_id(rb_cTrueClass, s_to_msgpack, MessagePack_TrueClass_to_msgpack, -1); rb_define_method_id(rb_cFalseClass, s_to_msgpack, MessagePack_FalseClass_to_msgpack, -1); rb_define_method_id(rb_cFixnum, s_to_msgpack, MessagePack_Fixnum_to_msgpack, -1); + rb_define_method_id(rb_cBignum, s_to_msgpack, MessagePack_Bignum_to_msgpack, -1); rb_define_method_id(rb_cFloat, s_to_msgpack, MessagePack_Float_to_msgpack, -1); rb_define_method_id(rb_cString, s_to_msgpack, MessagePack_String_to_msgpack, -1); rb_define_method_id(rb_cArray, s_to_msgpack, MessagePack_Array_to_msgpack, -1); diff --git a/ruby/unpack_inline.c b/ruby/unpack_inline.c index fa684c9..f6715d9 100644 --- a/ruby/unpack_inline.c +++ b/ruby/unpack_inline.c @@ -30,7 +30,7 @@ static inline VALUE msgpack_unpack_unsigned_int_32(msgpack_unpack_context* x, ui { return UINT2NUM(d); } static inline VALUE msgpack_unpack_unsigned_int_64(msgpack_unpack_context* x, uint64_t d) -{ return UINT2NUM(d); } // FIXME +{ return rb_ull2inum(d); } static inline VALUE msgpack_unpack_signed_int_8(msgpack_unpack_context* x, int8_t d) { return INT2FIX((long)d); } @@ -42,7 +42,7 @@ static inline VALUE msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int3 { return INT2NUM((long)d); } static inline VALUE msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d) -{ return INT2NUM(d); } // FIXME +{ return rb_ll2inum(d); } static inline VALUE msgpack_unpack_float(msgpack_unpack_context* x, float d) { return rb_float_new(d); } @@ -63,7 +63,7 @@ static inline VALUE msgpack_unpack_array_start(msgpack_unpack_context* x, unsign { return rb_ary_new2(n); } static inline void msgpack_unpack_array_item(msgpack_unpack_context* x, VALUE c, VALUE o) -{ rb_ary_push(c, o); } +{ rb_ary_push(c, o); } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] static inline VALUE msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n) { return rb_hash_new(); } From 529a50633dffc91dd5ce58ae5a905a0ac4a5fdf9 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0003/1172] lang/c/msgpack: added C++ binding git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@50 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/Makefile | 21 + cpp/msgpack | 1 + cpp/object.hpp | 760 ++++++++++++++++++++++++++++++++ cpp/test.cpp | 110 +++++ cpp/unpack.cc | 30 -- cpp/unpack.cpp | 96 ++++ cpp/unpack.h | 126 ------ cpp/unpack.hpp | 48 ++ cpp/unpack_context.h | 13 - cpp/unpack_context.hpp | 14 + cpp/unpack_inline.cpp | 72 +++ cpp/zone.cpp | 40 ++ cpp/zone.hpp.erb | 123 ++++++ msgpack/pack/inline_context.h | 2 - msgpack/pack/inline_impl.h | 66 +-- msgpack/unpack/inline_context.h | 7 + msgpack/unpack/inline_impl.h | 42 +- 17 files changed, 1361 insertions(+), 210 deletions(-) create mode 100644 cpp/Makefile create mode 120000 cpp/msgpack create mode 100644 cpp/object.hpp create mode 100644 cpp/test.cpp delete mode 100644 cpp/unpack.cc create mode 100644 cpp/unpack.cpp delete mode 100644 cpp/unpack.h create mode 100644 cpp/unpack.hpp delete mode 100644 cpp/unpack_context.h create mode 100644 cpp/unpack_context.hpp create mode 100644 cpp/unpack_inline.cpp create mode 100644 cpp/zone.cpp create mode 100644 cpp/zone.hpp.erb delete mode 100644 msgpack/pack/inline_context.h diff --git a/cpp/Makefile b/cpp/Makefile new file mode 100644 index 0000000..eafa683 --- /dev/null +++ b/cpp/Makefile @@ -0,0 +1,21 @@ + +CXXFLAGS = -I.. -I. -Wall -g -O4 +LDFLAGS = -L. + +NEED_PREPROCESS = zone.hpp + +all: test + +%.hpp: %.hpp.erb + erb $< > $@ + +test: $(NEED_PREPROCESS) unpack.o unpack_inline.o zone.o test.o object.hpp unpack.hpp + $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o test.o -o $@ + +.PHONY: clean +clean: + $(RM) unpack.o unpack_inline.o zone.o + $(RM) test.o + $(RM) test + $(RM) $(NEED_PREPROCESS) + diff --git a/cpp/msgpack b/cpp/msgpack new file mode 120000 index 0000000..945c9b4 --- /dev/null +++ b/cpp/msgpack @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/cpp/object.hpp b/cpp/object.hpp new file mode 100644 index 0000000..c4e94d7 --- /dev/null +++ b/cpp/object.hpp @@ -0,0 +1,760 @@ +#ifndef MSGPACK_OBJECT_HPP__ +#define MSGPACK_OBJECT_HPP__ +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace msgpack { + + +class type_error : public std::bad_cast { }; +class cast_error : public type_error { }; +class overflow_error : public type_error { }; +class underflow_error : public type_error { }; +class positive_overflow_error : public overflow_error { }; +class negative_overflow_error : public overflow_error { }; + + +struct raw { + explicit raw() : ptr(NULL), len(0) {} + explicit raw(void* p, size_t l) : ptr(p), len(l) {} +public: + void* ptr; + size_t len; +public: + std::string str() { return std::string((const char*)ptr, len); } +}; + +struct const_raw { + const_raw() : ptr(NULL), len(0) {} + const_raw(const void* p, size_t l) : ptr(p), len(l) {} +public: + const void* ptr; + size_t len; +public: + std::string str() { return std::string((const char*)ptr, len); } +}; + + +struct object; + +typedef std::map map; +typedef std::vector array; + + +template +struct numeric_overflow_signed_impl; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) < sizeof(X) ) { + if( static_cast( std::numeric_limits::max()) < x ) { return 1; } + if( static_cast(-std::numeric_limits::max()) > x ) { return -1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast( std::numeric_limits::max()) < x) { return 1; } + if( static_cast(-std::numeric_limits::max()) > x) { return -1; } + } + return 0; + } +}; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) <= sizeof(X) ) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast( std::numeric_limits::max()) < x) { return 1; } + } + return 0; + } +}; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( static_cast(0) > x ) { return -1; } + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) < sizeof(X) ) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast( std::numeric_limits::max()) < x) { return 1; } + } + return 0; + } +}; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) < sizeof(X) ) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + return 0; + } +}; + +template +struct numeric_overflow { + static int test(X x) { + return numeric_overflow_signed_impl::is_signed, std::numeric_limits::is_signed>::test(x); + } + static void check(X x) { + int r = test(x); + if(r == 1) { throw positive_overflow_error(); } + if(r == -1) { throw negative_overflow_error(); } + } +}; + + +template +struct numeric_underflow { + static bool test(X x) { + return static_cast(static_cast(x)) != x; + } + static void check(X x) { + if(test(x)) { throw underflow_error(); } + } +}; + + +struct object_class { + virtual ~object_class() {} + virtual bool isnil() const { return false; } + virtual bool xbool() const { throw cast_error(); } + virtual uint8_t xu8() const { throw cast_error(); } + virtual uint16_t xu16() const { throw cast_error(); } + virtual uint32_t xu32() const { throw cast_error(); } + virtual uint64_t xu64() const { throw cast_error(); } + virtual int8_t xi8() const { throw cast_error(); } + virtual int16_t xi16() const { throw cast_error(); } + virtual int32_t xi32() const { throw cast_error(); } + virtual int64_t xi64() const { throw cast_error(); } + virtual float xfloat() const { throw cast_error(); } + virtual double xdouble() const { throw cast_error(); } + virtual raw xraw() { throw cast_error(); } + virtual array& xarray() { throw cast_error(); } + virtual map& xmap() { throw cast_error(); } + virtual const_raw xraw() const { throw cast_error(); } + virtual const array& xarray() const { throw cast_error(); } + virtual const map& xmap() const { throw cast_error(); } + virtual bool operator== (const object_class* x) const { return false; } + bool operator!= (const object_class* x) const { return !(this->operator==(x)); } + virtual bool operator< (const object_class* x) const { throw cast_error(); } + virtual bool operator> (const object_class* x) const { throw cast_error(); } + operator bool() const { return xbool(); } // FIXME !isnil(); + operator uint8_t() const { return xu8(); } + operator uint16_t() const { return xu16(); } + operator uint32_t() const { return xu32(); } + operator uint64_t() const { return xu64(); } + operator int8_t() const { return xi8(); } + operator int16_t() const { return xi16(); } + operator int32_t() const { return xi32(); } + operator int64_t() const { return xi64(); } + operator float() const { return xfloat(); } + operator double() const { return xdouble(); } + operator raw() { return xraw(); } + operator array&() { return xarray(); } + operator map&() { return xmap(); } + operator const_raw() const { return xraw(); } + operator const array&() const { return xarray(); } + operator const map&() const { return xmap(); } + virtual const object_class* inspect(std::ostream& s) const + { s << '<' << typeid(*this).name() << '>'; return this; } +protected: + template + static void check_overflow(X x) { numeric_overflow::check(x); } + template + static void check_underflow(X x) { numeric_underflow::check(x); } +}; + +inline std::ostream& operator<< (std::ostream& s, const object_class* o) + { o->inspect(s); return s; } + + +struct object_container_mixin {}; +struct object_constructor_mixin {}; + + +struct object { + explicit object() : val(NULL) {} + object(object_class* v) : val(v) {} + //object(object_class& v) : val(&v) {} + ~object() {} + bool isnil() const { return val->isnil(); } + bool xbool() const { return val->xbool(); } + uint8_t xu8() const { return val->xu8(); } + uint16_t xu16() const { return val->xu16(); } + uint32_t xu32() const { return val->xu32(); } + uint64_t xu64() const { return val->xu64(); } + int8_t xi8() const { return val->xi8(); } + int16_t xi16() const { return val->xi16(); } + int32_t xi32() const { return val->xi32(); } + int64_t xi64() const { return val->xi64(); } + float xfloat() const { return val->xfloat(); } + double xdouble() const { return val->xdouble(); } + raw xraw() { return val->xraw(); } + array& xarray() { return val->xarray(); } + map& xmap() { return val->xmap(); } + const_raw xraw() const { return const_cast(val)->xraw(); } + const array& xarray() const { return const_cast(val)->xarray(); } + const map& xmap() const { return const_cast(val)->xmap(); } + bool operator== (object x) const { return val->operator== (x.val); } + bool operator!= (object x) const { return val->operator!= (x.val); } + bool operator< (object x) const { return val->operator< (x.val); } + bool operator> (object x) const { return val->operator> (x.val); } + operator bool() const { return val->operator bool(); } + operator uint8_t() const { return val->operator uint8_t(); } + operator uint16_t() const { return val->operator uint16_t(); } + operator uint32_t() const { return val->operator uint32_t(); } + operator uint64_t() const { return val->operator uint64_t(); } + operator int8_t() const { return val->operator int8_t(); } + operator int16_t() const { return val->operator int16_t(); } + operator int32_t() const { return val->operator int32_t(); } + operator int64_t() const { return val->operator int64_t(); } + operator float() const { return val->operator float(); } + operator double() const { return val->operator double(); } + operator raw() { return val->operator raw(); } + operator array&() { return val->operator array&(); } + operator map&() { return val->operator map&(); } + operator raw() const { return val->operator raw(); } + operator array&() const { return val->operator array&(); } + operator map&() const { return val->operator map&(); } + const object& inspect(std::ostream& s) const + { val->inspect(s); return *this; } +private: + friend class object_container_mixin; + friend class object_constructor_mixin; + object_class* val; +}; + +inline std::ostream& operator<< (std::ostream& s, const object& o) + { o.inspect(s); return s; } + + +struct object_nil : object_class { + bool isnil() const { return true; } + bool operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } + const object_class* inspect(std::ostream& s) const + { s << "nil"; return this; } +}; + +struct object_true : object_class { + bool xbool() const { return true; } + bool operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } + const object_class* inspect(std::ostream& s) const + { s << "true"; return this; } +}; + +struct object_false : object_class { + bool xbool() const { return false; } + bool operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } + const object_class* inspect(std::ostream& s) const + { s << "false"; return this; } +}; + +struct object_u8 : object_class { + explicit object_u8(uint8_t v) : val(v) {} + uint8_t xu8() const { return val; } + uint16_t xu16() const { return static_cast(val); } + uint32_t xu32() const { return static_cast(val); } + uint64_t xu64() const { return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { return static_cast(val); } + int32_t xi32() const { return static_cast(val); } + int64_t xi64() const { return static_cast(val); } + float xfloat() const { return static_cast(val); } + double xdouble() const { return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xu8(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xu8(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xu8(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << (uint16_t)val; return this; } +private: + uint8_t val; +}; + +struct object_u16 : object_class { + explicit object_u16(uint16_t v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { return val; } + uint32_t xu32() const { return static_cast(val); } + uint64_t xu64() const { return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { check_overflow(val); + return static_cast(val); } + int32_t xi32() const { return static_cast(val); } + int64_t xi64() const { return static_cast(val); } + float xfloat() const { return static_cast(val); } + double xdouble() const { return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xu16(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xu16(); } + catch (positive_overflow_error&) { return true; } + catch (type_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xu16(); } + catch (negative_overflow_error&) { return true; } + catch (type_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + uint16_t val; +}; + +struct object_u32 : object_class { + explicit object_u32(uint32_t v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { return val; } + uint64_t xu64() const { return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { check_overflow(val); + return static_cast(val); } + int32_t xi32() const { check_overflow(val); + return static_cast(val); } + int64_t xi64() const { return static_cast(val); } + float xfloat() const { check_underflow(val); + return static_cast(val); } + double xdouble() const { return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xu32(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xu32(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xu32(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + uint32_t val; +}; + +struct object_u64 : object_class { + explicit object_u64(uint64_t v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { check_overflow(val); + return static_cast(val); } + uint64_t xu64() const { return val; } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { check_overflow(val); + return static_cast(val); } + int32_t xi32() const { check_overflow(val); + return static_cast(val); } + int64_t xi64() const { check_overflow(val); + return static_cast(val); } + float xfloat() const { check_underflow(val); + return static_cast(val); } + double xdouble() const { check_underflow(val); + return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xu64(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xu64(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xu64(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + uint64_t val; +}; + +struct object_i8 : object_class { + explicit object_i8(int8_t v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { check_overflow(val); + return static_cast(val); } + uint64_t xu64() const { check_overflow(val); + return static_cast(val); } + int8_t xi8() const { return val; } + int16_t xi16() const { return static_cast(val); } + int32_t xi32() const { return static_cast(val); } + int64_t xi64() const { return static_cast(val); } + float xfloat() const { return static_cast(val); } + double xdouble() const { return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xi8(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xi8(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xi8(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << (int16_t)val; return this; } +private: + int8_t val; +}; + +struct object_i16 : object_class { + explicit object_i16(int16_t v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { check_overflow(val); + return static_cast(val); } + uint64_t xu64() const { check_overflow(val); + return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { return val; } + int32_t xi32() const { return static_cast(val); } + int64_t xi64() const { return static_cast(val); } + float xfloat() const { return static_cast(val); } + double xdouble() const { return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xi16(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xi16(); } + catch (positive_overflow_error&) { return true; } + catch (type_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xi16(); } + catch (negative_overflow_error&) { return true; } + catch (type_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + int16_t val; +}; + +struct object_i32 : object_class { + explicit object_i32(int32_t v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { check_overflow(val); + return static_cast(val); } + uint64_t xu64() const { check_overflow(val); + return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { check_overflow(val); + return static_cast(val); } + int32_t xi32() const { return val; } + int64_t xi64() const { return static_cast(val); } + float xfloat() const { check_underflow(val); + return static_cast(val); } + double xdouble() const { return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xi32(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xi32(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xi32(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + int32_t val; +}; + +struct object_i64 : object_class { + explicit object_i64(int64_t v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { check_overflow(val); + return static_cast(val); } + uint64_t xu64() const { check_overflow(val); + return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { check_overflow(val); + return static_cast(val); } + int32_t xi32() const { check_overflow(val); + return static_cast(val); } + int64_t xi64() const { return val; } + float xfloat() const { check_underflow(val); + return static_cast(val); } + double xdouble() const { check_underflow(val); + return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xi64(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xi64(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xi64(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + int64_t val; +}; + + +struct object_float : object_class { + object_float(float v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { check_overflow(val); + return static_cast(val); } + uint64_t xu64() const { check_overflow(val); + return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { check_overflow(val); + return static_cast(val); } + int32_t xi32() const { check_overflow(val); + return static_cast(val); } + int64_t xi64() const { check_overflow(val); + return static_cast(val); } + float xfloat() const { return val; } + double xdouble() const { return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xfloat(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return static_cast(val) < x->xdouble(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } + catch (underflow_error&) { + if(val < 0.0) { + if(numeric_overflow::test(val) == -1) { return true; } + try { return static_cast(val) < x->xi64(); } + catch (type_error&) { return true; } + } else { + if(numeric_overflow::test(val) == 1) { return false; } + try { return static_cast(val) < x->xu64(); } + catch (type_error&) { return false; } + } + } } + bool operator> (const object_class* x) const { try { return static_cast(val) > x->xdouble(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } + catch (underflow_error&) { + if(val < 0.0) { + if(numeric_overflow::test(val) == -1) { return false; } + try { return static_cast(val) > x->xi64(); } + catch (type_error&) { return false; } + } else { + if(numeric_overflow::test(val) == 1) { return true; } + try { return static_cast(val) > x->xu64(); } + catch (type_error&) { return true; } + } + } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + float val; +}; + + +struct object_double : object_class { + object_double(double v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { check_overflow(val); + return static_cast(val); } + uint64_t xu64() const { check_overflow(val); + return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { check_overflow(val); + return static_cast(val); } + int32_t xi32() const { check_overflow(val); + return static_cast(val); } + int64_t xi64() const { check_overflow(val); + return static_cast(val); } + float xfloat() const { check_overflow(val); + check_underflow(val); + return static_cast(val); } + double xdouble() const { return val; } + bool operator== (const object_class* x) const { try { return val == x->xdouble(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xdouble(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } + catch (underflow_error&) { + if(val < 0.0) { + if(numeric_overflow::test(val) == -1) { return true; } + try { return static_cast(val) < x->xi64(); } + catch (type_error&) { return true; } + } else { + if(numeric_overflow::test(val) == 1) { return false; } + try { return static_cast(val) < x->xu64(); } + catch (type_error&) { return false; } + } + } } + bool operator> (const object_class* x) const { try { return val > x->xdouble(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } + catch (underflow_error&) { + if(val < 0.0) { + if(numeric_overflow::test(val) == -1) { return false; } + try { return static_cast(val) > x->xi64(); } + catch (type_error&) { return false; } + } else { + if(numeric_overflow::test(val) == 1) { return true; } + try { return static_cast(val) > x->xu64(); } + catch (type_error&) { return true; } + } + } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + double val; +}; + + +struct object_raw : object_class { + explicit object_raw(void* p, uint32_t l) : ptr(p), len(l) {} + raw xraw() { return raw(ptr, len); } + const_raw xraw() const { return const_raw(ptr, len); } + bool operator== (const object_class* x) const { try { const_raw xr(x->xraw()); + return len == xr.len && (ptr == xr.ptr || memcmp(ptr, xr.ptr, len) == 0); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { const_raw xr(x->xraw()); + if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) < 0; } + else { return len < xr.len; } } + bool operator> (const object_class* x) const { const_raw xr(x->xraw()); + if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } + else { return len > xr.len; } } + const object_class* inspect(std::ostream& s) const + { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape +private: + void* ptr; + uint32_t len; +}; + +struct object_const_raw : object_class { + explicit object_const_raw(const void* p, uint32_t l) : ptr(p), len(l) {} + const_raw xraw() const { return const_raw(ptr, len); } + bool operator== (const object_class* x) const { try { const_raw xr(x->xraw()); + return len == xr.len && (ptr == xr.ptr || memcmp(ptr, xr.ptr, len) == 0); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { const_raw xr(x->xraw()); + if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) < 0; } + else { return len < xr.len; } } + bool operator> (const object_class* x) const { const_raw xr(x->xraw()); + if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } + else { return len > xr.len; } } + const object_class* inspect(std::ostream& s) const + { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape +private: + const void* ptr; + uint32_t len; +}; + +struct object_array : object_class, object_container_mixin { + explicit object_array() {} + explicit object_array(uint32_t n) { val.reserve(n); } + array& xarray() { return val; } + const array& xarray() const { return val; } + bool operator== (const object_class* x) const { try { + const std::vector& xa(x->xarray()); + if(val.size() != xa.size()) { return false; } + for(std::vector::const_iterator iv(val.begin()), iv_end(val.end()), ix(xa.begin()); + iv != iv_end; + ++iv, ++ix) { + if(*iv != *ix) { return false; } + } + return true; + } catch (type_error&) { return false; } } + // FIXME operator< operator> + const object_class* inspect(std::ostream& s) const { + s << '['; + if(!val.empty()) { + std::vector::const_iterator it(val.begin()); + s << *it; + ++it; + for(std::vector::const_iterator it_end(val.end()); + it != it_end; + ++it) { + s << ", " << *it; + } + } + s << ']'; + return this; } +public: + void push_back(object o) { val.push_back(o); } +private: + std::vector val; +}; + +// FIXME hash, operator==: nil, true, false, containerを入れられãªã„ +struct object_map : object_class, object_container_mixin { + explicit object_map() {} + map& xmap() { return val; } + const map& xmap() const { return val; } + bool operator== (const object_class* x) const { try { + const std::map& xm(x->xmap()); + if(val.size() != xm.size()) { return false; } + for(std::map::const_iterator iv(val.begin()), iv_end(val.end()), ix(xm.begin()); + iv != iv_end; + ++iv, ++ix) { + if(iv->first != ix->first || iv->second != ix->first) { return false; } + } + return true; + } catch (type_error&) { return false; } } + // FIXME operator< operator> + const object_class* inspect(std::ostream& s) const { + s << '{'; + if(!val.empty()) { + std::map::const_iterator it(val.begin()); + s << it->first << "=>" << it->second; + ++it; + for(std::map::const_iterator it_end(val.end()); + it != it_end; + ++it) { + s << ", " << it->first << "=>" << it->second; + } + } + s << '}'; + return this; } +public: + void store(object k, object v) { val[k] = v; } +private: + std::map val; +}; + + +} // namespace msgpack + +#endif /* msgpack/object.hpp */ + diff --git a/cpp/test.cpp b/cpp/test.cpp new file mode 100644 index 0000000..423a6bd --- /dev/null +++ b/cpp/test.cpp @@ -0,0 +1,110 @@ +#include +#include + +class checker { +public: + void check(const char* d, size_t len, msgpack::object should) { + try { + std::cout << "----" << std::endl; + msgpack::object o; + try { + o = msgpack::unpack(d, len, m_zone); + } catch (std::runtime_error& e) { + std::cout << should << std::endl; + std::cout << "**" << e.what() << "**" << std::endl; + return; + } + std::cout << o << std::endl; + if(o != should) { + std::cout << "** TEST FAILED **" << std::endl; + } + } catch (...) { m_zone.clear(); throw; } + m_zone.clear(); + } +private: + msgpack::zone m_zone; +}; + +int main(void) +{ + +checker c; + +{ // SimpleValue + msgpack::zone z; + const char d[] = { + 0x93, 0xc0, 0xc2, 0xc3, + }; + c.check(d, sizeof(d), + z.narray( + z.nnil(), z.nfalse(), z.ntrue() + ) + ); +} + +{ // Fixnum + msgpack::zone z; + const char d[] = { + 0x92, + 0x93, 0x00, 0x40, 0x7f, + 0x93, 0xe0, 0xf0, 0xff, + }; + c.check(d, sizeof(d), + z.narray( + z.narray( + z.nu8(0), + z.nu8(64), + z.nu8(127) + ), + z.narray( + z.ni8(-32), + z.ni8(-16), + z.ni8(-1) + ) + ) + ); +} + +{ // FixArray + msgpack::zone z; + const char d[] = { + 0x92, + 0x90, + 0x91, + 0x91, 0xc0, + }; + c.check(d, sizeof(d), + z.narray( + z.narray(), + z.narray( + z.narray( + z.nnil() + ) + ) + ) + ); +} + +{ // FixRaw + msgpack::zone z; + const char d[] = { + 0x94, + 0xa0, + 0xa1, 'a', + 0xa2, 'b', 'c', + 0xa3, 'd', 'e', 'f', + }; + c.check(d, sizeof(d), + z.narray( + z.nraw("", 0), + z.nraw("a", 1), + z.nraw("bc", 2), + z.nraw("def", 3) + ) + ); +} + + +return 0; +} + diff --git a/cpp/unpack.cc b/cpp/unpack.cc deleted file mode 100644 index ce57e67..0000000 --- a/cpp/unpack.cc +++ /dev/null @@ -1,30 +0,0 @@ -#include "unpack.h" -#include "unpack_context.h" -#include - -msgpack_unpack_t* msgpack_unpack_new(void) -{ - msgpack_unpacker* ctx; - ctx = (msgpack_unpacker*)calloc(1, sizeof(msgpack_unpacker)); - if(ctx == NULL) { return NULL; } - msgpack_unpacker_init(ctx); - return (msgpack_unpack_t*)ctx; -} - -void msgpack_unpack_free(msgpack_unpack_t* ctx) -{ - free((msgpack_unpacker*)ctx); -} - -int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off) -{ - return msgpack_unpacker_execute( - (msgpack_unpacker*)ctx, - data, len, off); -} - -void* msgpack_unpack_data(msgpack_unpack_t* ctx) -{ - return msgpack_unpacker_data((msgpack_unpacker*)ctx); -} - diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp new file mode 100644 index 0000000..0f02d3c --- /dev/null +++ b/cpp/unpack.cpp @@ -0,0 +1,96 @@ +#include "msgpack/unpack.hpp" +#include "unpack_context.hpp" +#include + +namespace msgpack { + +struct unpacker::context { + context(zone& z) + { + msgpack_unpacker_init(&m_ctx); + m_ctx.user = &z; + } + + ~context() { } + + int execute(const void* data, size_t len, size_t* off) + { + return msgpack_unpacker_execute(&m_ctx, (const char*)data, len, off); + } + + object_class* data() + { + return msgpack_unpacker_data(&m_ctx); + } + + void reset() + { + zone* z = m_ctx.user; + msgpack_unpacker_init(&m_ctx); + m_ctx.user = z; + } + +private: + msgpack_unpacker m_ctx; + +private: + context(); + context(const context&); +}; + + +unpacker::unpacker(zone& z) : + m_ctx(new context(z)), + m_zone(z), + m_finished(false) +{ } + + +unpacker::~unpacker() { delete m_ctx; } + + +size_t unpacker::execute(const void* data, size_t len, size_t off) +{ + int ret = m_ctx->execute(data, len, &off); + if(ret < 0) { + throw unpack_error("parse error"); + } else if(ret > 0) { + m_finished = true; + return off; + } else { + m_finished = false; + return off; + } +} + + +object unpacker::data() +{ + return object(m_ctx->data()); +} + + +void unpacker::reset() +{ + m_ctx->reset(); +} + + +object unpacker::unpack(const void* data, size_t len, zone& z) +{ + context ctx(z); + size_t off = 0; + int ret = ctx.execute(data, len, &off); + if(ret < 0) { + throw unpack_error("parse error"); + } else if(ret == 0) { + throw unpack_error("insufficient bytes"); + } else if(off < len) { + throw unpack_error("extra bytes"); + } + return ctx.data(); +} + + +} // namespace msgpack + diff --git a/cpp/unpack.h b/cpp/unpack.h deleted file mode 100644 index cf7a168..0000000 --- a/cpp/unpack.h +++ /dev/null @@ -1,126 +0,0 @@ -#ifndef MSGPACK_UNPACK_H__ -#define MSGPACK_UNPACK_H__ - -#include -#include - -namespace MessagePack { - -class Unpacker { - class object { - template - object(const T& x) : m_obj(new holder(x)) {} - }; - - class type_error : public std::exception { }; - class cast_error : public type_error { }; - class overflow_error : public type_error { }; - class underflow_error : public type_error { }; - - struct object { - virtual ~object() {} - virtual bool isnil() const { return false; } - virtual bool xbool() const { throw cast_error(); } - virtual uint8_t xu8() const { throw cast_error(); } - virtual uint16_t xu16() const { throw cast_error(); } - virtual uint32_t xu32() const { throw cast_error(); } - virtual uint64_t xu64() const { throw cast_error(); } - virtual int8_t xi8() const { throw cast_error(); } - virtual int16_t xi16() const { throw cast_error(); } - virtual int32_t xi32() const { throw cast_error(); } - virtual int64_t xi64() const { throw cast_error(); } - virtual float xfloat() const { throw cast_error(); } - virtual double xdouble() const { throw cast_error(); } - virtual std::map& xmap() const { throw cast_error(); } - virtual std::string& xstring() const { throw cast_error(); } - virtual std::pair xraw() const { throw cast_error(); } - public: - template - inline void check_overflow(X x) { - if(std::numeric_limits::max() < x) { throw overflow_error(); } - } - template - inline void check_underflow(X x) { - if(std::numeric_limits::min() > x) { throw overflow_error(); } - } - }; - -private: - struct object_nil : object { - bool isnil() const { return true; } - }; - - struct object_true : object { - bool xbool() const { return true; } - }; - - struct object_false : object { - bool xbool() const { return false; } - }; - - struct object_u8 : object { - object_u8(uint8_t val) : m_val(val) {} - uint8_t xu8() const { return m_val; } - uint16_t xu16() const { return static_cast(m_val); } - uint32_t xu32() const { return static_cast(m_val); } - uint64_t xu64() const { return static_cast(m_val); } - int8_t xi8() const { check_overflow(m_val); return m_val; } - int16_t xi16() const { return static_cast(m_val); } - int32_t xi32() const { return static_cast(m_val); } - int64_t xi64() const { return static_cast(m_val); } - private: - uint8_t m_val; - }; - - struct object_u16 : object { - object_u16(uint16_t val) : m_val(val) {} - uint8_t xu8() const { check_overflow(m_val); return m_val; } - uint16_t xu16() const { return m_val; } - uint32_t xu32() const { return static_cast(m_val); } - uint64_t xu64() const { return static_cast(m_val); } - int8_t xi8() const { check_overflow< int8_t>(m_val); return m_val; } - int16_t xi16() const { check_overflow(m_val); return m_val; } - int32_t xi32() const { return static_cast(m_val); } - int64_t xi64() const { return static_cast(m_val); } - private: - uint16_t m_val; - }; - - ... -}; - -} // namespace MessagePack - -typedef struct { - void* (*unpack_unsigned_int_8)(void* data, uint8_t d); - void* (*unpack_unsigned_int_16)(void* data, uint16_t d); - void* (*unpack_unsigned_int_32)(void* data, uint32_t d); - void* (*unpack_unsigned_int_64)(void* data, uint64_t d); - void* (*unpack_signed_int_8)(void* data, int8_t d); - void* (*unpack_signed_int_16)(void* data, int16_t d); - void* (*unpack_signed_int_32)(void* data, int32_t d); - void* (*unpack_signed_int_64)(void* data, int64_t d); - void* (*unpack_float)(void* data, float d); - void* (*unpack_double)(void* data, double d); - void* (*unpack_big_int)(void* data, const void* b, unsigned int l); - void* (*unpack_big_float)(void* data, const void* b, unsigned int l); - void* (*unpack_nil)(void* data); - void* (*unpack_true)(void* data); - void* (*unpack_false)(void* data); - void* (*unpack_array_start)(void* data, unsigned int n); - void (*unpack_array_item)(void* data, void* c, void* o); - void* (*unpack_map_start)(void* data, unsigned int n); - void (*unpack_map_item)(void* data, void* c, void* k, void* v); - void* (*unpack_string)(void* data, const void* b, size_t l); - void* (*unpack_raw)(void* data, const void* b, size_t l); - void* data; -} msgpack_unpack_t; - -msgpack_unpack_t* msgpack_unpack_new(void); -void msgpack_unpack_free(msgpack_unpack_t* ctx); -int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off); -void* msgpack_unpack_data(msgpack_unpack_t* ctx); - -#endif /* msgpack/unpack.h */ - - diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp new file mode 100644 index 0000000..61ba781 --- /dev/null +++ b/cpp/unpack.hpp @@ -0,0 +1,48 @@ +#ifndef MSGPACK_UNPACK_HPP__ +#define MSGPACK_UNPACK_HPP__ + +#include "msgpack/object.hpp" +#include "msgpack/zone.hpp" +#include + +namespace msgpack { + + +struct unpack_error : public std::runtime_error { + unpack_error(const std::string& msg) : + std::runtime_error(msg) { } +}; + + +class unpacker { +public: + unpacker(zone& z); + ~unpacker(); +public: + size_t execute(const void* data, size_t len, size_t off); + bool is_finished() { return m_finished; } + object data(); + void reset(); +private: + struct context; + context* m_ctx; + zone& m_zone; + bool m_finished; +private: + unpacker(); + unpacker(const unpacker&); +public: + static object unpack(const void* data, size_t len, zone& z); +}; + + +inline object unpack(const void* data, size_t len, zone& z) +{ + return unpacker::unpack(data, len, z); +} + + +} // namespace msgpack + +#endif /* msgpack/unpack.hpp */ + diff --git a/cpp/unpack_context.h b/cpp/unpack_context.h deleted file mode 100644 index caf2271..0000000 --- a/cpp/unpack_context.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef UNPACK_CONTEXT_H__ -#define UNPACK_CONTEXT_H__ - -#include "unpack.h" - -typedef void* msgpack_object; - -typedef msgpack_unpack_t msgpack_unpack_context; - -#include "msgpack/unpack/inline_context.h" - -#endif /* unpack_context.h */ - diff --git a/cpp/unpack_context.hpp b/cpp/unpack_context.hpp new file mode 100644 index 0000000..49a93e4 --- /dev/null +++ b/cpp/unpack_context.hpp @@ -0,0 +1,14 @@ +#ifndef UNPACK_CONTEXT_HPP__ +#define UNPACK_CONTEXT_HPP__ + +#include "msgpack/zone.hpp" +#include "msgpack/object.hpp" + +typedef msgpack::object_class* msgpack_object; + +typedef msgpack::zone* msgpack_unpack_context; + +#include "msgpack/unpack/inline_context.h" + +#endif /* unpack_context.h */ + diff --git a/cpp/unpack_inline.cpp b/cpp/unpack_inline.cpp new file mode 100644 index 0000000..82f6e7a --- /dev/null +++ b/cpp/unpack_inline.cpp @@ -0,0 +1,72 @@ +#include "unpack_context.hpp" + + +extern "C" { +using namespace msgpack; + + +static inline object_class* msgpack_unpack_init(zone** z) +{ return NULL; } + +static inline object_class* msgpack_unpack_unsigned_int_8(zone** z, uint8_t d) +{ return (*z)->nu8(d); } + +static inline object_class* msgpack_unpack_unsigned_int_16(zone** z, uint16_t d) +{ return (*z)->nu16(d); } + +static inline object_class* msgpack_unpack_unsigned_int_32(zone** z, uint32_t d) +{ return (*z)->nu32(d); } + +static inline object_class* msgpack_unpack_unsigned_int_64(zone** z, uint64_t d) +{ return (*z)->nu64(d); } + +static inline object_class* msgpack_unpack_signed_int_8(zone** z, int8_t d) +{ return (*z)->ni8(d); } + +static inline object_class* msgpack_unpack_signed_int_16(zone** z, int16_t d) +{ return (*z)->ni16(d); } + +static inline object_class* msgpack_unpack_signed_int_32(zone** z, int32_t d) +{ return (*z)->ni32(d); } + +static inline object_class* msgpack_unpack_signed_int_64(zone** z, int64_t d) +{ return (*z)->ni64(d); } + +static inline object_class* msgpack_unpack_float(zone** z, float d) +{ return (*z)->nfloat(d); } + +static inline object_class* msgpack_unpack_double(zone** z, double d) +{ return (*z)->ndouble(d); } + +static inline object_class* msgpack_unpack_nil(zone** z) +{ return (*z)->nnil(); } + +static inline object_class* msgpack_unpack_true(zone** z) +{ return (*z)->ntrue(); } + +static inline object_class* msgpack_unpack_false(zone** z) +{ return (*z)->nfalse(); } + +static inline object_class* msgpack_unpack_array_start(zone** z, unsigned int n) +{ return (*z)->narray(n); } + +static inline void msgpack_unpack_array_item(zone** z, object_class* c, object_class* o) +{ reinterpret_cast(c)->push_back(o); } + +static inline object_class* msgpack_unpack_map_start(zone** z, unsigned int n) +{ return (*z)->narray(); } + +static inline void msgpack_unpack_map_item(zone** z, object_class* c, object_class* k, object_class* v) +{ reinterpret_cast(c)->store(k, v); } + +static inline object_class* msgpack_unpack_string(zone** z, const void* b, size_t l) +{ return (*z)->nraw(b, l); } + +static inline object_class* msgpack_unpack_raw(zone** z, const void* b, size_t l) +{ return (*z)->nraw(b, l); } + + +} // extern "C" + +#include "msgpack/unpack/inline_impl.h" + diff --git a/cpp/zone.cpp b/cpp/zone.cpp new file mode 100644 index 0000000..5031467 --- /dev/null +++ b/cpp/zone.cpp @@ -0,0 +1,40 @@ +#include "zone.hpp" + +namespace msgpack { + + +void* zone::alloc() +{ + if(m_used >= m_pool.size()*MSGPACK_ZONE_CHUNK_SIZE) { + m_pool.push_back(chunk_t()); + } + void* data = m_pool[m_used/MSGPACK_ZONE_CHUNK_SIZE].cells[m_used%MSGPACK_ZONE_CHUNK_SIZE].data; + ++m_used; + return data; +} + +void zone::clear() +{ + for(size_t b=0; b < m_used/MSGPACK_ZONE_CHUNK_SIZE; ++b) { + cell_t* c(m_pool[b].cells); + for(size_t e=0; e < MSGPACK_ZONE_CHUNK_SIZE; ++e) { + reinterpret_cast(c[e].data)->~object_class(); + } + } + cell_t* c(m_pool.back().cells); + for(size_t e=0; e < m_used%MSGPACK_ZONE_CHUNK_SIZE; ++e) { + reinterpret_cast(c[e].data)->~object_class(); + } + m_used = 0; + m_pool.resize(1); + for(user_finalizer_t::reverse_iterator it(m_user_finalizer.rbegin()), it_end(m_user_finalizer.rend()); + it != it_end; + ++it) { + it->call(); + } + m_user_finalizer.clear(); +} + + +} // namespace msgpack + diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb new file mode 100644 index 0000000..1a941af --- /dev/null +++ b/cpp/zone.hpp.erb @@ -0,0 +1,123 @@ +#ifndef MSGPACK_ZONE_HPP__ +#define MSGPACK_ZONE_HPP__ + +#include "msgpack/object.hpp" +#include + +#ifndef MSGPACK_ZONE_CHUNK_SIZE +#define MSGPACK_ZONE_CHUNK_SIZE 64 +#endif + +namespace msgpack { + + +class zone { +public: +zone() : m_used(0), m_pool(1) { } +~zone() { clear(); } + +public: + template + void push_finalizer(void (*func)(void* obj, void* user), T* obj, void* user); + +public: + object_nil* nnil() { return new (alloc()) object_nil(); } + object_true* ntrue() { return new (alloc()) object_true(); } + object_false* nfalse() { return new (alloc()) object_false(); } + object_u8* nu8( uint8_t v) { return new (alloc()) object_u8(v); } + object_u16* nu16(uint16_t v) { return new (alloc()) object_u16(v); } + object_u32* nu32(uint32_t v) { return new (alloc()) object_u32(v); } + object_u64* nu64(uint64_t v) { return new (alloc()) object_u64(v); } + object_i8* ni8( int8_t v) { return new (alloc()) object_i8(v); } + object_i16* ni16( int16_t v) { return new (alloc()) object_i16(v); } + object_i32* ni32( int32_t v) { return new (alloc()) object_i32(v); } + object_i64* ni64( int64_t v) { return new (alloc()) object_i64(v); } + object_float* nfloat( float v) { return new (alloc()) object_float(v); } + object_double* ndouble( double v) { return new (alloc()) object_double(v); } + + object_raw* nraw(void* ptr, uint32_t len) + { return new (alloc()) object_raw(ptr, len); } + + object_const_raw* nraw(const void* ptr, uint32_t len) + { return new (alloc()) object_const_raw(ptr, len); } + + object_array* narray() + { return new (alloc()) object_array(); } + + object_array* narray(size_t reserve_size) + { return new (alloc()) object_array(reserve_size); } + + object_map* nmap() + { return new (alloc()) object_map(); } + +<% GENERATION_SIZE = 16 %> +<% 1.upto(GENERATION_SIZE) {|i| %> + object_array* narray(<% 1.upto(i-1) {|n| %>object o<%=n%>, <% } %>object o<%=i%>) + { object_array* a = new (alloc()) object_array(<%=i%>); + <% 1.upto(i) {|n| %>a->push_back(o<%=n%>); + <% } %>return a; } +<% } %> + +<% 1.upto(GENERATION_SIZE) {|i| %> + object_map* nmap(<% 1.upto(i-1) {|n| %>object k<%=n%>, object v<%=n%>, <% } %>object k<%=i%>, object v<%=i%>) + { object_map* m = new (alloc()) object_map(); + <% 1.upto(i) {|n| %>m->store(k<%=n%>, v<%=n%>); + <% } %>return m; } +<% } %> + +public: + void clear(); + +private: + void* alloc(); + +private: + size_t m_used; + + static const size_t MAX_OBJECT_SIZE = + sizeof(object_raw) > sizeof(object_array) + ? ( sizeof(object_raw) > sizeof(object_map) + ? sizeof(object_raw) + : sizeof(object_map) + ) + : ( sizeof(object_array) > sizeof(object_map) + ? sizeof(object_array) + : sizeof(object_map) + ) + ; + + struct cell_t { + char data[MAX_OBJECT_SIZE]; + }; + + struct chunk_t { + cell_t cells[MSGPACK_ZONE_CHUNK_SIZE]; + }; + + typedef std::vector pool_t; + pool_t m_pool; + + + class finalizer { + public: + finalizer(void (*func)(void*, void*), void* obj, void* user) : + m_obj(obj), m_user(user), m_func(func) {} + void call() { (*m_func)(m_obj, m_user); } + private: + void* m_obj; + void* m_user; + void (*m_func)(void*, void*); + }; + + typedef std::vector user_finalizer_t; + user_finalizer_t m_user_finalizer; + +private: + zone(const zone&); +}; + + +} // namespace msgpack + +#endif /* msgpack/zone.hpp */ + diff --git a/msgpack/pack/inline_context.h b/msgpack/pack/inline_context.h deleted file mode 100644 index 139597f..0000000 --- a/msgpack/pack/inline_context.h +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/msgpack/pack/inline_impl.h b/msgpack/pack/inline_impl.h index 5c4bfed..08a5bc1 100644 --- a/msgpack/pack/inline_impl.h +++ b/msgpack/pack/inline_impl.h @@ -23,25 +23,25 @@ #ifdef __LITTLE_ENDIAN__ -#define STORE_16(d) \ +#define STORE_BE16(d) \ ((char*)&d)[1], ((char*)&d)[0] -#define STORE_32(d) \ +#define STORE_BE32(d) \ ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] -#define STORE_64(d) \ +#define STORE_BE64(d) \ ((char*)&d)[7], ((char*)&d)[6], ((char*)&d)[5], ((char*)&d)[4], \ ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] #elif __BIG_ENDIAN__ -#define STORE_16(d) \ - ((char*)&d)[2], ((char*)&d)[3] +#define STORE_BE16(d) \ + ((char*)&d)[0], ((char*)&d)[1] -#define STORE_32(d) \ +#define STORE_BE32(d) \ ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3] -#define STORE_32(d) \ +#define STORE_BE64(d) \ ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3], \ ((char*)&d)[4], ((char*)&d)[5], ((char*)&d)[6], ((char*)&d)[7] @@ -57,10 +57,10 @@ inline void msgpack_pack_int(msgpack_pack_context x, int d) { if(d < -32) { if(d < -32768) { // signed 32 - const unsigned char buf[5] = {0xd2, STORE_32(d)}; + const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } else if(d < -128) { // signed 16 - const unsigned char buf[3] = {0xd1, STORE_16(d)}; + const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { // signed 8 const unsigned char buf[2] = {0xd0, (uint8_t)d}; @@ -75,11 +75,11 @@ inline void msgpack_pack_int(msgpack_pack_context x, int d) msgpack_pack_append_buffer(x, buf, 2); } else if(d < 65536) { // unsigned 16 - const unsigned char buf[3] = {0xcd, STORE_16(d)}; + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { // unsigned 32 - const unsigned char buf[5] = {0xce, STORE_32(d)}; + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -97,11 +97,11 @@ inline void msgpack_pack_unsigned_int(msgpack_pack_context x, unsigned int d) msgpack_pack_append_buffer(x, buf, 2); } else if(d < 65536) { // unsigned 16 - const unsigned char buf[3] = {0xcd, STORE_16(d)}; + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { // unsigned 32 - const unsigned char buf[5] = {0xce, STORE_32(d)}; + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -118,20 +118,20 @@ inline void msgpack_pack_unsigned_int_8(msgpack_pack_context x, uint8_t d) inline void msgpack_pack_unsigned_int_16(msgpack_pack_context x, uint16_t d) { - const unsigned char buf[3] = {0xcd, STORE_16(d)}; + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } inline void msgpack_pack_unsigned_int_32(msgpack_pack_context x, uint32_t d) { - const unsigned char buf[5] = {0xce, STORE_32(d)}; + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } inline void msgpack_pack_unsigned_int_64(msgpack_pack_context x, uint64_t d) { - // FIXME - const unsigned char buf[9] = {0xcf, STORE_64(d)}; + // FIXME optimization + const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -149,20 +149,20 @@ inline void msgpack_pack_signed_int_8(msgpack_pack_context x, int8_t d) inline void msgpack_pack_signed_int_16(msgpack_pack_context x, int16_t d) { - const unsigned char buf[3] = {0xd1, STORE_16(d)}; + const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } inline void msgpack_pack_signed_int_32(msgpack_pack_context x, int32_t d) { - const unsigned char buf[5] = {0xd2, STORE_32(d)}; + const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } inline void msgpack_pack_signed_int_64(msgpack_pack_context x, int64_t d) { - // FIXME - const unsigned char buf[9] = {0xd3, STORE_64(d)}; + // FIXME optimization + const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -174,14 +174,14 @@ inline void msgpack_pack_signed_int_64(msgpack_pack_context x, int64_t d) inline void msgpack_pack_float(msgpack_pack_context x, float d) { uint32_t n = *((uint32_t*)&d); // FIXME - const unsigned char buf[5] = {0xca, STORE_32(n)}; + const unsigned char buf[5] = {0xca, STORE_BE32(n)}; msgpack_pack_append_buffer(x, buf, 5); } inline void msgpack_pack_double(msgpack_pack_context x, double d) { uint64_t n = *((uint64_t*)&d); // FIXME - const unsigned char buf[9] = {0xcb, STORE_64(n)}; + const unsigned char buf[9] = {0xcb, STORE_BE64(n)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -200,6 +200,7 @@ inline void msgpack_pack_nil(msgpack_pack_context x) /* * Boolean */ + inline void msgpack_pack_true(msgpack_pack_context x) { static const unsigned char d = 0xc3; @@ -224,11 +225,11 @@ inline void msgpack_pack_array(msgpack_pack_context x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xdc, STORE_16(d)}; + unsigned char buf[3] = {0xdc, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdd, STORE_32(d)}; + unsigned char buf[5] = {0xdd, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -245,18 +246,18 @@ inline void msgpack_pack_map(msgpack_pack_context x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xde, STORE_16(d)}; + unsigned char buf[3] = {0xde, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdf, STORE_32(d)}; + unsigned char buf[5] = {0xdf, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } /* - * String + * Raw */ inline void msgpack_pack_string(msgpack_pack_context x, const char* b) @@ -272,16 +273,21 @@ inline void msgpack_pack_raw(msgpack_pack_context x, const void* b, size_t l) msgpack_pack_append_buffer(x, &d, 1); } else if(l < 65536) { uint16_t d = (uint16_t)l; - unsigned char buf[3] = {0xda, STORE_16(d)}; + unsigned char buf[3] = {0xda, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)l; - unsigned char buf[5] = {0xdb, STORE_32(d)}; + unsigned char buf[5] = {0xdb, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_append_buffer(x, b, l); } +#undef STORE_BE16(d) +#undef STORE_BE32(d) +#undef STORE_BE64(d) + + #endif /* msgpack/pack/inline_impl.h */ diff --git a/msgpack/unpack/inline_context.h b/msgpack/unpack/inline_context.h index aecd566..d6558a3 100644 --- a/msgpack/unpack/inline_context.h +++ b/msgpack/unpack/inline_context.h @@ -25,6 +25,10 @@ #define MSG_STACK_SIZE 16 #endif +#ifdef __cplusplus +extern "C" { +#endif + typedef struct { msgpack_object obj; size_t count; @@ -47,6 +51,9 @@ void msgpack_unpacker_init(msgpack_unpacker* ctx); int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len, size_t* off); #define msgpack_unpacker_data(unpacker) (unpacker)->stack[0].obj +#ifdef __cplusplus +} +#endif #endif /* msgpack/unpack/inline_context.h */ diff --git a/msgpack/unpack/inline_impl.h b/msgpack/unpack/inline_impl.h index ec7f0fc..f610dd7 100644 --- a/msgpack/unpack/inline_impl.h +++ b/msgpack/unpack/inline_impl.h @@ -23,6 +23,10 @@ #include /*#include */ +#ifdef __cplusplus +extern "C" { +#endif + // Positive FixNum 0xxxxxxx 0x00 - 0x7f // Negative FixNum 111xxxxx 0xe0 - 0xff // Variable 110xxxxx 0xc0 - 0xdf @@ -71,13 +75,16 @@ #endif #endif -static inline uint64_t ntohll(uint64_t x) { -#ifdef __LITTLE_ENDIAN__ // FIXME +#define betoh16(x) ntohs(x) +#define betoh32(x) ntohl(x) + +#ifdef __LITTLE_ENDIAN__ #if defined(__bswap_64) - return __bswap_64(x); +# define betoh64(x) __bswap_64(x) #elif defined(__DARWIN_OSSwapInt64) - return __DARWIN_OSSwapInt64(x); +# define betoh64(x) __DARWIN_OSSwapInt64(x) #else +static inline uint64_t betoh64(uint64_t x) { return ((x << 56) & 0xff00000000000000ULL ) | ((x << 40) & 0x00ff000000000000ULL ) | ((x << 24) & 0x0000ff0000000000ULL ) | @@ -86,11 +93,12 @@ static inline uint64_t ntohll(uint64_t x) { ((x >> 24) & 0x0000000000ff0000ULL ) | ((x >> 40) & 0x000000000000ff00ULL ) | ((x >> 56) & 0x00000000000000ffULL ) ; +} #endif #else - return x; +#define betoh64(x) (x) #endif -} + typedef enum { CS_HEADER = 0x00, // nil @@ -212,9 +220,9 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len ((unsigned int)*p & 0x1f) #define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) ntohs(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) ntohl(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) ntohll(*(uint64_t*)ptr) +#define PTR_CAST_16(ptr) betoh16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) betoh32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) betoh64(*(uint64_t*)ptr) if(p == pe) { goto _out; } do { @@ -434,5 +442,21 @@ _end: } +#ifdef betoh16(x) +#undef betoh16(x) +#endif + +#ifdef betoh32(x) +#undef betoh32(x) +#endif + +#ifdef betoh64(x) +#undef betoh64(x) +#endif + +#ifdef __cplusplus +} +#endif + #endif /* msgpack/unpack/inline_impl.h */ From 7c427400a7071f0cb251edd102430945e142033f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0004/1172] lang/c/msgpack: update C++ code git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@51 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/bench.c | 8 +- cpp/Makefile | 6 +- cpp/object.cpp | 340 ++++++++++++++++++++++++++ cpp/object.hpp | 652 +++++++------------------------------------------ 4 files changed, 439 insertions(+), 567 deletions(-) create mode 100644 cpp/object.cpp diff --git a/c/bench.c b/c/bench.c index f27350a..0f316dc 100644 --- a/c/bench.c +++ b/c/bench.c @@ -98,9 +98,9 @@ static void pack_append_buffer(void* user, const unsigned char* b, unsigned int static const unsigned int TASK_INT_NUM = 1<<24; -static const unsigned int TASK_STR_LEN = 1<<15; +//static const unsigned int TASK_STR_LEN = 1<<15; //static const unsigned int TASK_INT_NUM = 1<<20; -//static const unsigned int TASK_STR_LEN = 1<<12; +static const unsigned int TASK_STR_LEN = 1<<12; static const char* TASK_STR_PTR; @@ -284,6 +284,7 @@ void bench_msgpack(void) msgpack_unpack_reset(mupk); + /* puts("----"); puts("pack string"); reset_timer(); @@ -316,6 +317,7 @@ void bench_msgpack(void) sec = show_timer(); printf("%f MB/s\n", len / sec / 1024 / 1024); + */ msgpack_unpack_free(mupk); @@ -330,7 +332,7 @@ int main(int argc, char* argv[]) TASK_STR_PTR = str; bench_msgpack(); - bench_json(); +// bench_json(); return 0; } diff --git a/cpp/Makefile b/cpp/Makefile index eafa683..04b421c 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -9,12 +9,12 @@ all: test %.hpp: %.hpp.erb erb $< > $@ -test: $(NEED_PREPROCESS) unpack.o unpack_inline.o zone.o test.o object.hpp unpack.hpp - $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o test.o -o $@ +test: $(NEED_PREPROCESS) unpack.o unpack_inline.o object.o zone.o test.o object.hpp unpack.hpp + $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o object.o test.o -o $@ .PHONY: clean clean: - $(RM) unpack.o unpack_inline.o zone.o + $(RM) unpack.o unpack_inline.o object.o zone.o $(RM) test.o $(RM) test $(RM) $(NEED_PREPROCESS) diff --git a/cpp/object.cpp b/cpp/object.cpp new file mode 100644 index 0000000..099b541 --- /dev/null +++ b/cpp/object.cpp @@ -0,0 +1,340 @@ +#include "msgpack/object.hpp" + +namespace msgpack { + +namespace { + +template +struct numeric_overflow_signed_impl; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) < sizeof(X) ) { + if( static_cast( std::numeric_limits::max()) < x ) { return 1; } + if( static_cast(-std::numeric_limits::max()) > x ) { return -1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast( std::numeric_limits::max()) < x) { return 1; } + if( static_cast(-std::numeric_limits::max()) > x) { return -1; } + } + return 0; + } +}; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) <= sizeof(X) ) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast( std::numeric_limits::max()) < x) { return 1; } + } + return 0; + } +}; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( static_cast(0) > x ) { return -1; } + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) < sizeof(X) ) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast( std::numeric_limits::max()) < x) { return 1; } + } + return 0; + } +}; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) < sizeof(X) ) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + return 0; + } +}; + +template +struct numeric_overflow { + static int test(X x) { + return numeric_overflow_signed_impl::is_signed, std::numeric_limits::is_signed>::test(x); + } + static void check(X x) { + int r = test(x); + if(r == 1) { throw positive_overflow_error(); } + if(r == -1) { throw negative_overflow_error(); } + } +}; + +template +struct numeric_underflow { + static bool test(X x) { + return static_cast(static_cast(x)) != x; + } + static void check(X x) { + if(test(x)) { throw underflow_error(); } + } +}; + +template +inline T integer_cast(X x) { + numeric_overflow::check(x); + return static_cast(x); } + +template +inline T float_cast(X x) { + numeric_overflow::check(x); + numeric_underflow::check(x); + return static_cast(x); } + +template +inline bool numequal(V v, const object_class* x) + try { return v == static_cast(*x); } + catch (type_error&) { return false; } + +template +inline bool numless(V v, const object_class* x) + try { return v < static_cast(*x); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } + +template +inline bool numgreater(V v, const object_class* x) + try { return v > static_cast(*x); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } + +template +inline void numeric_inspect(V v, std::ostream& s) + { s << v; } + +template <> +inline void numeric_inspect(uint8_t v, std::ostream& s) + { s << (uint16_t)v; } + +template <> +inline void numeric_inspect(int8_t v, std::ostream& s) + { s << (int16_t)v; } + +} // noname namespace + + +bool object_nil::isnil() const { return true; } +bool object_nil::operator== (const object_class* x) const + { return typeid(*this) == typeid(*x); } +const object_class* object_nil::inspect(std::ostream& s) const + { s << "nil"; return this; } + +bool object_true::xbool() const { return true; } +bool object_true::operator== (const object_class* x) const + { return typeid(*this) == typeid(*x); } +const object_class* object_true::inspect(std::ostream& s) const + { s << "true"; return this; } + +bool object_false::xbool() const { return false; } +bool object_false::operator== (const object_class* x) const + { return typeid(*this) == typeid(*x); } +const object_class* object_false::inspect(std::ostream& s) const + { s << "false"; return this; } + + +#define INTEGER_OBJECT(NAME) \ +uint8_t object_##NAME::xu8 () const { return val; } \ +uint16_t object_##NAME::xu16 () const { return integer_cast(val); } \ +uint32_t object_##NAME::xu32 () const { return integer_cast(val); } \ +uint64_t object_##NAME::xu64 () const { return integer_cast(val); } \ +int8_t object_##NAME::xi8 () const { return integer_cast(val); } \ +int16_t object_##NAME::xi16 () const { return integer_cast(val); } \ +int32_t object_##NAME::xi32 () const { return integer_cast(val); } \ +int64_t object_##NAME::xi64 () const { return integer_cast(val); } \ +float object_##NAME::xfloat () const { return integer_cast(val); } \ +double object_##NAME::xdouble() const { return integer_cast(val); } \ +bool object_##NAME::operator== (const object_class* x) const \ + try { return val == x->x##NAME(); } \ + catch (type_error&) { return false; } \ +bool object_##NAME::operator< (const object_class* x) const \ + try { return val < x->x##NAME(); } \ + catch (positive_overflow_error&) { return true; } \ + catch (overflow_error&) { return false; } \ +bool object_##NAME::operator> (const object_class* x) const \ + try { return val > x->x##NAME(); } \ + catch (negative_overflow_error&) { return true; } \ + catch (overflow_error&) { return false; } \ +const object_class* object_##NAME::inspect(std::ostream& s) const \ + { numeric_inspect(val, s); return this; } \ + + +INTEGER_OBJECT(u8) +INTEGER_OBJECT(u16) +INTEGER_OBJECT(u32) +INTEGER_OBJECT(u64) +INTEGER_OBJECT(i8) +INTEGER_OBJECT(i16) +INTEGER_OBJECT(i32) +INTEGER_OBJECT(i64) + +#undef INTEGER_OBJECT(NAME) + + +#define FLOAT_OBJECT(NAME) \ +uint8_t object_##NAME::xu8 () const { return val; } \ +uint16_t object_##NAME::xu16 () const { return integer_cast(val); } \ +uint32_t object_##NAME::xu32 () const { return integer_cast(val); } \ +uint64_t object_##NAME::xu64 () const { return integer_cast(val); } \ +int8_t object_##NAME::xi8 () const { return integer_cast(val); } \ +int16_t object_##NAME::xi16 () const { return integer_cast(val); } \ +int32_t object_##NAME::xi32 () const { return integer_cast(val); } \ +int64_t object_##NAME::xi64 () const { return integer_cast(val); } \ +float object_##NAME::xfloat () const { return float_cast(val); } \ +double object_##NAME::xdouble() const { return float_cast(val); } \ +bool object_##NAME::operator== (const object_class* x) const \ + try { return val == x->x##NAME(); } \ + catch (type_error&) { return false; } \ +bool object_##NAME::operator< (const object_class* x) const { \ + try { return val < x->xdouble(); } \ + catch (positive_overflow_error&) { return true; } \ + catch (overflow_error&) { return false; } \ + catch (underflow_error&) { \ + if(val < 0.0) { \ + if(numeric_overflow::test(val) == -1) { return true; } \ + try { return static_cast(val) < x->xi64(); } \ + catch (type_error&) { return true; } \ + } else { \ + if(numeric_overflow::test(val) == 1) { return false; } \ + try { return static_cast(val) < x->xu64(); } \ + catch (type_error&) { return false; } \ + } \ + } } \ +bool object_##NAME::operator> (const object_class* x) const { \ + try { return val > x->xdouble(); } \ + catch (negative_overflow_error&) { return true; } \ + catch (overflow_error&) { return false; } \ + catch (underflow_error&) { \ + if(val < 0.0) { \ + if(numeric_overflow::test(val) == -1) { return false; } \ + try { return static_cast(val) > x->xi64(); } \ + catch (type_error&) { return false; } \ + } else { \ + if(numeric_overflow::test(val) == 1) { return true; } \ + try { return static_cast(val) > x->xu64(); } \ + catch (type_error&) { return true; } \ + } \ + } } \ +const object_class* object_##NAME::inspect(std::ostream& s) const \ + { s << val; return this; } \ + +FLOAT_OBJECT(float) +FLOAT_OBJECT(double) + +#undef FLOAT_OBJECT(NAME) + + +#define RAW_OBJECT(NAME, EXTRA) \ +EXTRA \ +bool object_##NAME::operator== (const object_class* x) const \ + try { \ + const_raw xr(x->xraw()); \ + return len == xr.len && (ptr == xr.ptr || memcmp(ptr, xr.ptr, len) == 0); \ + } catch (type_error&) { return false; } \ +bool object_##NAME::operator< (const object_class* x) const { \ + const_raw xr(x->xraw()); \ + if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) < 0; } \ + else { return len < xr.len; } } \ +bool object_##NAME::operator> (const object_class* x) const { \ + const_raw xr(x->xraw()); \ + if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } \ + else { return len > xr.len; } } \ +const object_class* object_##NAME::inspect(std::ostream& s) const \ + { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape + + +RAW_OBJECT(raw, + raw object_raw::xraw() { return raw(ptr, len); } + const_raw object_raw::xraw() const { return const_raw(ptr, len); } ) + +RAW_OBJECT(const_raw, + const_raw object_const_raw::xraw() const { return const_raw(ptr, len); } ) + +#undef RAW_OBJECT(NAME, EXTRA) + + + array& object_array::xarray() { return val; } +const array& object_array::xarray() const { return val; } +bool object_array::operator== (const object_class* x) const + try { + const std::vector& xa(x->xarray()); + if(val.size() != xa.size()) { return false; } + for(std::vector::const_iterator iv(val.begin()), iv_end(val.end()), ix(xa.begin()); + iv != iv_end; + ++iv, ++ix) { + if(*iv != *ix) { return false; } + } + return true; + } catch (type_error&) { return false; } +const object_class* object_array::inspect(std::ostream& s) const +{ + s << '['; + if(!val.empty()) { + std::vector::const_iterator it(val.begin()); + s << *it; + ++it; + for(std::vector::const_iterator it_end(val.end()); + it != it_end; + ++it) { + s << ", " << *it; + } + } + s << ']'; + return this; +} + + + map& object_map::xmap() { return val; } +const map& object_map::xmap() const { return val; } +bool object_map::operator== (const object_class* x) const + try { + const std::map& xm(x->xmap()); + if(val.size() != xm.size()) { return false; } + for(std::map::const_iterator iv(val.begin()), iv_end(val.end()), ix(xm.begin()); + iv != iv_end; + ++iv, ++ix) { + if(iv->first != ix->first || iv->second != ix->first) { return false; } + } + return true; + } catch (type_error&) { return false; } +const object_class* object_map::inspect(std::ostream& s) const +{ + s << '{'; + if(!val.empty()) { + std::map::const_iterator it(val.begin()); + s << it->first << "=>" << it->second; + ++it; + for(std::map::const_iterator it_end(val.end()); + it != it_end; + ++it) { + s << ", " << it->first << "=>" << it->second; + } + } + s << '}'; + return this; +} + + +} // namespace msgpack + diff --git a/cpp/object.hpp b/cpp/object.hpp index c4e94d7..992ac1e 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -48,96 +48,6 @@ typedef std::map map; typedef std::vector array; -template -struct numeric_overflow_signed_impl; - -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) < sizeof(X) ) { - if( static_cast( std::numeric_limits::max()) < x ) { return 1; } - if( static_cast(-std::numeric_limits::max()) > x ) { return -1; } - } - } else if(std::numeric_limits::is_integer) { - if( static_cast( std::numeric_limits::max()) < x) { return 1; } - if( static_cast(-std::numeric_limits::max()) > x) { return -1; } - } - return 0; - } -}; - -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) <= sizeof(X) ) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } - } - } else if(std::numeric_limits::is_integer) { - if( static_cast( std::numeric_limits::max()) < x) { return 1; } - } - return 0; - } -}; - -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( static_cast(0) > x ) { return -1; } - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) < sizeof(X) ) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } - } - } else if(std::numeric_limits::is_integer) { - if( static_cast( std::numeric_limits::max()) < x) { return 1; } - } - return 0; - } -}; - -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) < sizeof(X) ) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } - } - } else if(std::numeric_limits::is_integer) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } - } - return 0; - } -}; - -template -struct numeric_overflow { - static int test(X x) { - return numeric_overflow_signed_impl::is_signed, std::numeric_limits::is_signed>::test(x); - } - static void check(X x) { - int r = test(x); - if(r == 1) { throw positive_overflow_error(); } - if(r == -1) { throw negative_overflow_error(); } - } -}; - - -template -struct numeric_underflow { - static bool test(X x) { - return static_cast(static_cast(x)) != x; - } - static void check(X x) { - if(test(x)) { throw underflow_error(); } - } -}; - - struct object_class { virtual ~object_class() {} virtual bool isnil() const { return false; } @@ -181,11 +91,6 @@ struct object_class { operator const map&() const { return xmap(); } virtual const object_class* inspect(std::ostream& s) const { s << '<' << typeid(*this).name() << '>'; return this; } -protected: - template - static void check_overflow(X x) { numeric_overflow::check(x); } - template - static void check_underflow(X x) { numeric_underflow::check(x); } }; inline std::ostream& operator<< (std::ostream& s, const object_class* o) @@ -253,500 +158,125 @@ inline std::ostream& operator<< (std::ostream& s, const object& o) struct object_nil : object_class { - bool isnil() const { return true; } - bool operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } - const object_class* inspect(std::ostream& s) const - { s << "nil"; return this; } + bool isnil() const; + bool operator== (const object_class* x) const; + const object_class* inspect(std::ostream& s) const; }; struct object_true : object_class { - bool xbool() const { return true; } - bool operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } - const object_class* inspect(std::ostream& s) const - { s << "true"; return this; } + bool xbool() const; + bool operator== (const object_class* x) const; + const object_class* inspect(std::ostream& s) const; }; struct object_false : object_class { - bool xbool() const { return false; } - bool operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } - const object_class* inspect(std::ostream& s) const - { s << "false"; return this; } + bool xbool() const; + bool operator== (const object_class* x) const; + const object_class* inspect(std::ostream& s) const; }; -struct object_u8 : object_class { - explicit object_u8(uint8_t v) : val(v) {} - uint8_t xu8() const { return val; } - uint16_t xu16() const { return static_cast(val); } - uint32_t xu32() const { return static_cast(val); } - uint64_t xu64() const { return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { return static_cast(val); } - int32_t xi32() const { return static_cast(val); } - int64_t xi64() const { return static_cast(val); } - float xfloat() const { return static_cast(val); } - double xdouble() const { return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xu8(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xu8(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xu8(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << (uint16_t)val; return this; } -private: - uint8_t val; +#define INTEGER_CLASS(TYPE, NAME) \ +struct object_##NAME : object_class { \ + explicit object_##NAME(TYPE v) : val(v) {} \ + uint8_t xu8 () const; \ + uint16_t xu16 () const; \ + uint32_t xu32 () const; \ + uint64_t xu64 () const; \ + int8_t xi8 () const; \ + int16_t xi16 () const; \ + int32_t xi32 () const; \ + int64_t xi64 () const; \ + float xfloat () const; \ + double xdouble() const; \ + bool operator== (const object_class* x) const; \ + bool operator< (const object_class* x) const; \ + bool operator> (const object_class* x) const; \ + const object_class* inspect(std::ostream& s) const; \ +private: \ + TYPE val; \ }; -struct object_u16 : object_class { - explicit object_u16(uint16_t v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { return val; } - uint32_t xu32() const { return static_cast(val); } - uint64_t xu64() const { return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { check_overflow(val); - return static_cast(val); } - int32_t xi32() const { return static_cast(val); } - int64_t xi64() const { return static_cast(val); } - float xfloat() const { return static_cast(val); } - double xdouble() const { return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xu16(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xu16(); } - catch (positive_overflow_error&) { return true; } - catch (type_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xu16(); } - catch (negative_overflow_error&) { return true; } - catch (type_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - uint16_t val; +INTEGER_CLASS(uint8_t, u8) +INTEGER_CLASS(uint16_t, u16) +INTEGER_CLASS(uint32_t, u32) +INTEGER_CLASS(uint64_t, u64) +INTEGER_CLASS(int8_t, i8) +INTEGER_CLASS(int16_t, i16) +INTEGER_CLASS(int32_t, i32) +INTEGER_CLASS(int64_t, i64) + +#undef INTEGER_CLASS(TYPE, NAME) + + +#define FLOAT_CLASS(TYPE, NAME) \ +struct object_##NAME : object_class { \ + object_##NAME(TYPE v) : val(v) {} \ + uint8_t xu8 () const; \ + uint16_t xu16 () const; \ + uint32_t xu32 () const; \ + uint64_t xu64 () const; \ + int8_t xi8 () const; \ + int16_t xi16 () const; \ + int32_t xi32 () const; \ + int64_t xi64 () const; \ + float xfloat () const; \ + double xdouble() const; \ + bool operator== (const object_class* x) const; \ + bool operator< (const object_class* x) const; \ + bool operator> (const object_class* x) const; \ + const object_class* inspect(std::ostream& s) const; \ +private: \ + TYPE val; \ }; -struct object_u32 : object_class { - explicit object_u32(uint32_t v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { return val; } - uint64_t xu64() const { return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { check_overflow(val); - return static_cast(val); } - int32_t xi32() const { check_overflow(val); - return static_cast(val); } - int64_t xi64() const { return static_cast(val); } - float xfloat() const { check_underflow(val); - return static_cast(val); } - double xdouble() const { return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xu32(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xu32(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xu32(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - uint32_t val; +FLOAT_CLASS(float, float) +FLOAT_CLASS(double, double) + +#undef FLOAT_CLASS(TYPE, NAME) + + +#define RAW_CLASS(NAME, TYPE, EXTRA) \ +struct object_##NAME : object_class { \ + explicit object_##NAME(TYPE p, uint32_t l) : ptr(p), len(l) {} \ + EXTRA \ + bool operator== (const object_class* x) const; \ + bool operator< (const object_class* x) const; \ + bool operator> (const object_class* x) const; \ + const object_class* inspect(std::ostream& s) const; \ +private: \ + TYPE ptr; \ + uint32_t len; \ }; -struct object_u64 : object_class { - explicit object_u64(uint64_t v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { check_overflow(val); - return static_cast(val); } - uint64_t xu64() const { return val; } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { check_overflow(val); - return static_cast(val); } - int32_t xi32() const { check_overflow(val); - return static_cast(val); } - int64_t xi64() const { check_overflow(val); - return static_cast(val); } - float xfloat() const { check_underflow(val); - return static_cast(val); } - double xdouble() const { check_underflow(val); - return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xu64(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xu64(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xu64(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - uint64_t val; -}; +RAW_CLASS(raw, void*, raw xraw(); const_raw xraw() const; ) +RAW_CLASS(const_raw, const void*, const_raw xraw() const; ) -struct object_i8 : object_class { - explicit object_i8(int8_t v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { check_overflow(val); - return static_cast(val); } - uint64_t xu64() const { check_overflow(val); - return static_cast(val); } - int8_t xi8() const { return val; } - int16_t xi16() const { return static_cast(val); } - int32_t xi32() const { return static_cast(val); } - int64_t xi64() const { return static_cast(val); } - float xfloat() const { return static_cast(val); } - double xdouble() const { return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xi8(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xi8(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xi8(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << (int16_t)val; return this; } -private: - int8_t val; -}; +#undef RAW_CLASS(NAME, TYPE, EXTRA) -struct object_i16 : object_class { - explicit object_i16(int16_t v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { check_overflow(val); - return static_cast(val); } - uint64_t xu64() const { check_overflow(val); - return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { return val; } - int32_t xi32() const { return static_cast(val); } - int64_t xi64() const { return static_cast(val); } - float xfloat() const { return static_cast(val); } - double xdouble() const { return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xi16(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xi16(); } - catch (positive_overflow_error&) { return true; } - catch (type_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xi16(); } - catch (negative_overflow_error&) { return true; } - catch (type_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - int16_t val; -}; - -struct object_i32 : object_class { - explicit object_i32(int32_t v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { check_overflow(val); - return static_cast(val); } - uint64_t xu64() const { check_overflow(val); - return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { check_overflow(val); - return static_cast(val); } - int32_t xi32() const { return val; } - int64_t xi64() const { return static_cast(val); } - float xfloat() const { check_underflow(val); - return static_cast(val); } - double xdouble() const { return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xi32(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xi32(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xi32(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - int32_t val; -}; - -struct object_i64 : object_class { - explicit object_i64(int64_t v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { check_overflow(val); - return static_cast(val); } - uint64_t xu64() const { check_overflow(val); - return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { check_overflow(val); - return static_cast(val); } - int32_t xi32() const { check_overflow(val); - return static_cast(val); } - int64_t xi64() const { return val; } - float xfloat() const { check_underflow(val); - return static_cast(val); } - double xdouble() const { check_underflow(val); - return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xi64(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xi64(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xi64(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - int64_t val; -}; - - -struct object_float : object_class { - object_float(float v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { check_overflow(val); - return static_cast(val); } - uint64_t xu64() const { check_overflow(val); - return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { check_overflow(val); - return static_cast(val); } - int32_t xi32() const { check_overflow(val); - return static_cast(val); } - int64_t xi64() const { check_overflow(val); - return static_cast(val); } - float xfloat() const { return val; } - double xdouble() const { return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xfloat(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return static_cast(val) < x->xdouble(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } - catch (underflow_error&) { - if(val < 0.0) { - if(numeric_overflow::test(val) == -1) { return true; } - try { return static_cast(val) < x->xi64(); } - catch (type_error&) { return true; } - } else { - if(numeric_overflow::test(val) == 1) { return false; } - try { return static_cast(val) < x->xu64(); } - catch (type_error&) { return false; } - } - } } - bool operator> (const object_class* x) const { try { return static_cast(val) > x->xdouble(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } - catch (underflow_error&) { - if(val < 0.0) { - if(numeric_overflow::test(val) == -1) { return false; } - try { return static_cast(val) > x->xi64(); } - catch (type_error&) { return false; } - } else { - if(numeric_overflow::test(val) == 1) { return true; } - try { return static_cast(val) > x->xu64(); } - catch (type_error&) { return true; } - } - } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - float val; -}; - - -struct object_double : object_class { - object_double(double v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { check_overflow(val); - return static_cast(val); } - uint64_t xu64() const { check_overflow(val); - return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { check_overflow(val); - return static_cast(val); } - int32_t xi32() const { check_overflow(val); - return static_cast(val); } - int64_t xi64() const { check_overflow(val); - return static_cast(val); } - float xfloat() const { check_overflow(val); - check_underflow(val); - return static_cast(val); } - double xdouble() const { return val; } - bool operator== (const object_class* x) const { try { return val == x->xdouble(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xdouble(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } - catch (underflow_error&) { - if(val < 0.0) { - if(numeric_overflow::test(val) == -1) { return true; } - try { return static_cast(val) < x->xi64(); } - catch (type_error&) { return true; } - } else { - if(numeric_overflow::test(val) == 1) { return false; } - try { return static_cast(val) < x->xu64(); } - catch (type_error&) { return false; } - } - } } - bool operator> (const object_class* x) const { try { return val > x->xdouble(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } - catch (underflow_error&) { - if(val < 0.0) { - if(numeric_overflow::test(val) == -1) { return false; } - try { return static_cast(val) > x->xi64(); } - catch (type_error&) { return false; } - } else { - if(numeric_overflow::test(val) == 1) { return true; } - try { return static_cast(val) > x->xu64(); } - catch (type_error&) { return true; } - } - } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - double val; -}; - - -struct object_raw : object_class { - explicit object_raw(void* p, uint32_t l) : ptr(p), len(l) {} - raw xraw() { return raw(ptr, len); } - const_raw xraw() const { return const_raw(ptr, len); } - bool operator== (const object_class* x) const { try { const_raw xr(x->xraw()); - return len == xr.len && (ptr == xr.ptr || memcmp(ptr, xr.ptr, len) == 0); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { const_raw xr(x->xraw()); - if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) < 0; } - else { return len < xr.len; } } - bool operator> (const object_class* x) const { const_raw xr(x->xraw()); - if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } - else { return len > xr.len; } } - const object_class* inspect(std::ostream& s) const - { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape -private: - void* ptr; - uint32_t len; -}; - -struct object_const_raw : object_class { - explicit object_const_raw(const void* p, uint32_t l) : ptr(p), len(l) {} - const_raw xraw() const { return const_raw(ptr, len); } - bool operator== (const object_class* x) const { try { const_raw xr(x->xraw()); - return len == xr.len && (ptr == xr.ptr || memcmp(ptr, xr.ptr, len) == 0); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { const_raw xr(x->xraw()); - if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) < 0; } - else { return len < xr.len; } } - bool operator> (const object_class* x) const { const_raw xr(x->xraw()); - if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } - else { return len > xr.len; } } - const object_class* inspect(std::ostream& s) const - { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape -private: - const void* ptr; - uint32_t len; -}; struct object_array : object_class, object_container_mixin { explicit object_array() {} explicit object_array(uint32_t n) { val.reserve(n); } - array& xarray() { return val; } - const array& xarray() const { return val; } - bool operator== (const object_class* x) const { try { - const std::vector& xa(x->xarray()); - if(val.size() != xa.size()) { return false; } - for(std::vector::const_iterator iv(val.begin()), iv_end(val.end()), ix(xa.begin()); - iv != iv_end; - ++iv, ++ix) { - if(*iv != *ix) { return false; } - } - return true; - } catch (type_error&) { return false; } } - // FIXME operator< operator> - const object_class* inspect(std::ostream& s) const { - s << '['; - if(!val.empty()) { - std::vector::const_iterator it(val.begin()); - s << *it; - ++it; - for(std::vector::const_iterator it_end(val.end()); - it != it_end; - ++it) { - s << ", " << *it; - } - } - s << ']'; - return this; } + array& xarray(); + const array& xarray() const; + bool operator== (const object_class* x) const; + // FIXME operator<, operator> + const object_class* inspect(std::ostream& s) const; public: void push_back(object o) { val.push_back(o); } private: std::vector val; }; -// FIXME hash, operator==: nil, true, false, containerを入れられãªã„ + +// FIXME hash, operator==: nil, true, false, array, mapを入れられãªã„ struct object_map : object_class, object_container_mixin { explicit object_map() {} - map& xmap() { return val; } - const map& xmap() const { return val; } - bool operator== (const object_class* x) const { try { - const std::map& xm(x->xmap()); - if(val.size() != xm.size()) { return false; } - for(std::map::const_iterator iv(val.begin()), iv_end(val.end()), ix(xm.begin()); - iv != iv_end; - ++iv, ++ix) { - if(iv->first != ix->first || iv->second != ix->first) { return false; } - } - return true; - } catch (type_error&) { return false; } } - // FIXME operator< operator> - const object_class* inspect(std::ostream& s) const { - s << '{'; - if(!val.empty()) { - std::map::const_iterator it(val.begin()); - s << it->first << "=>" << it->second; - ++it; - for(std::map::const_iterator it_end(val.end()); - it != it_end; - ++it) { - s << ", " << it->first << "=>" << it->second; - } - } - s << '}'; - return this; } + map& xmap(); + const map& xmap() const; + bool operator== (const object_class* x) const; + // FIXME operator<, operator> + const object_class* inspect(std::ostream& s) const; public: void store(object k, object v) { val[k] = v; } private: From 9b95875d85fb802f6dd0fc1fe0544bef4be1fbc5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0005/1172] lang/c/msgpack: reverted c/bench.c git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@52 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/bench.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/c/bench.c b/c/bench.c index 0f316dc..f27350a 100644 --- a/c/bench.c +++ b/c/bench.c @@ -98,9 +98,9 @@ static void pack_append_buffer(void* user, const unsigned char* b, unsigned int static const unsigned int TASK_INT_NUM = 1<<24; -//static const unsigned int TASK_STR_LEN = 1<<15; +static const unsigned int TASK_STR_LEN = 1<<15; //static const unsigned int TASK_INT_NUM = 1<<20; -static const unsigned int TASK_STR_LEN = 1<<12; +//static const unsigned int TASK_STR_LEN = 1<<12; static const char* TASK_STR_PTR; @@ -284,7 +284,6 @@ void bench_msgpack(void) msgpack_unpack_reset(mupk); - /* puts("----"); puts("pack string"); reset_timer(); @@ -317,7 +316,6 @@ void bench_msgpack(void) sec = show_timer(); printf("%f MB/s\n", len / sec / 1024 / 1024); - */ msgpack_unpack_free(mupk); @@ -332,7 +330,7 @@ int main(int argc, char* argv[]) TASK_STR_PTR = str; bench_msgpack(); -// bench_json(); + bench_json(); return 0; } From f41c20a2503411393012d01e3f3c3ea0e9c7dde8 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0006/1172] lang/c/msgpack: added C++ binding msgpack::pack git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@53 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/pack_inline.h | 3 +- cpp/Makefile | 2 +- cpp/object.cpp | 77 ++++++++++++++- cpp/object.hpp | 49 ++++++---- cpp/pack.hpp | 192 +++++++++++++++++++++++++++++++++++++ cpp/test.cpp | 161 +++++++++++++++++-------------- msgpack/pack/inline_impl.h | 58 ++++++----- ruby/pack_inline.h | 2 + 8 files changed, 426 insertions(+), 118 deletions(-) create mode 100644 cpp/pack.hpp diff --git a/c/pack_inline.h b/c/pack_inline.h index d943464..b1f9e63 100644 --- a/c/pack_inline.h +++ b/c/pack_inline.h @@ -7,8 +7,9 @@ typedef msgpack_pack_t* msgpack_pack_context; static inline void msgpack_pack_append_buffer(msgpack_pack_t* x, const unsigned char* b, unsigned int l); +#include +#include /* __BYTE_ORDER */ #include "msgpack/pack/inline_impl.h" - #endif /* pack_inline.h */ diff --git a/cpp/Makefile b/cpp/Makefile index 04b421c..5ef7ecd 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -9,7 +9,7 @@ all: test %.hpp: %.hpp.erb erb $< > $@ -test: $(NEED_PREPROCESS) unpack.o unpack_inline.o object.o zone.o test.o object.hpp unpack.hpp +test: $(NEED_PREPROCESS) unpack.o unpack_inline.o object.o zone.o test.o object.hpp unpack.hpp pack.hpp $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o object.o test.o -o $@ .PHONY: clean diff --git a/cpp/object.cpp b/cpp/object.cpp index 099b541..02e0438 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -1,4 +1,5 @@ #include "msgpack/object.hpp" +#include "msgpack/pack.hpp" namespace msgpack { @@ -132,24 +133,73 @@ template <> inline void numeric_inspect(int8_t v, std::ostream& s) { s << (int16_t)v; } +template +inline void numeric_pack(dynamic_packer& p, V v); + +template <> +inline void numeric_pack(dynamic_packer& p, uint8_t v) + { p.pack_unsigned_int_8(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, uint16_t v) + { p.pack_unsigned_int_16(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, uint32_t v) + { p.pack_unsigned_int_32(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, uint64_t v) + { p.pack_unsigned_int_64(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, int8_t v) + { p.pack_unsigned_int_8(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, int16_t v) + { p.pack_unsigned_int_16(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, int32_t v) + { p.pack_unsigned_int_32(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, int64_t v) + { p.pack_unsigned_int_64(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, float v) + { p.pack_float(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, double v) + { p.pack_double(v); } + } // noname namespace bool object_nil::isnil() const { return true; } bool object_nil::operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } +void object_nil::pack(dynamic_packer& p) const + { p.pack_nil(); } const object_class* object_nil::inspect(std::ostream& s) const { s << "nil"; return this; } bool object_true::xbool() const { return true; } bool object_true::operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } +void object_true::pack(dynamic_packer& p) const + { p.pack_true(); } const object_class* object_true::inspect(std::ostream& s) const { s << "true"; return this; } bool object_false::xbool() const { return false; } bool object_false::operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } +void object_false::pack(dynamic_packer& p) const + { p.pack_false(); } const object_class* object_false::inspect(std::ostream& s) const { s << "false"; return this; } @@ -176,6 +226,8 @@ bool object_##NAME::operator> (const object_class* x) const \ try { return val > x->x##NAME(); } \ catch (negative_overflow_error&) { return true; } \ catch (overflow_error&) { return false; } \ +void object_##NAME::pack(dynamic_packer& p) const \ + { numeric_pack(p, val); } \ const object_class* object_##NAME::inspect(std::ostream& s) const \ { numeric_inspect(val, s); return this; } \ @@ -236,7 +288,9 @@ bool object_##NAME::operator> (const object_class* x) const { \ catch (type_error&) { return true; } \ } \ } } \ -const object_class* object_##NAME::inspect(std::ostream& s) const \ +void object_##NAME::pack(dynamic_packer& p) const \ + { numeric_pack(p, val); } \ +const object_class* object_##NAME::inspect(std::ostream& s) const \ { s << val; return this; } \ FLOAT_OBJECT(float) @@ -260,6 +314,8 @@ bool object_##NAME::operator> (const object_class* x) const { \ const_raw xr(x->xraw()); \ if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } \ else { return len > xr.len; } } \ +void object_##NAME::pack(dynamic_packer& p) const \ + { p.pack_raw(ptr, len); } \ const object_class* object_##NAME::inspect(std::ostream& s) const \ { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape @@ -303,6 +359,15 @@ const object_class* object_array::inspect(std::ostream& s) const s << ']'; return this; } +void object_array::pack(dynamic_packer& p) const +{ + p.pack_array(val.size()); + for(std::vector::const_iterator it(val.begin()), it_end(val.end()); + it != it_end; + ++it) { + it->pack(p); + } +} map& object_map::xmap() { return val; } @@ -334,6 +399,16 @@ const object_class* object_map::inspect(std::ostream& s) const s << '}'; return this; } +void object_map::pack(dynamic_packer& p) const +{ + p.pack_map(val.size()); + for(std::map::const_iterator it(val.begin()), it_end(val.end()); + it != it_end; + ++it) { + it->first.pack(p); + it->second.pack(p); + } +} } // namespace msgpack diff --git a/cpp/object.hpp b/cpp/object.hpp index 992ac1e..3f22dfc 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -1,6 +1,5 @@ #ifndef MSGPACK_OBJECT_HPP__ #define MSGPACK_OBJECT_HPP__ -#include #include #include @@ -47,6 +46,8 @@ struct object; typedef std::map map; typedef std::vector array; +class dynamic_packer; + struct object_class { virtual ~object_class() {} @@ -72,6 +73,7 @@ struct object_class { bool operator!= (const object_class* x) const { return !(this->operator==(x)); } virtual bool operator< (const object_class* x) const { throw cast_error(); } virtual bool operator> (const object_class* x) const { throw cast_error(); } + virtual void pack(dynamic_packer& p) const = 0; operator bool() const { return xbool(); } // FIXME !isnil(); operator uint8_t() const { return xu8(); } operator uint16_t() const { return xu16(); } @@ -128,6 +130,7 @@ struct object { bool operator!= (object x) const { return val->operator!= (x.val); } bool operator< (object x) const { return val->operator< (x.val); } bool operator> (object x) const { return val->operator> (x.val); } + void pack(dynamic_packer& p) const { val->pack(p); } operator bool() const { return val->operator bool(); } operator uint8_t() const { return val->operator uint8_t(); } operator uint16_t() const { return val->operator uint16_t(); } @@ -160,18 +163,21 @@ inline std::ostream& operator<< (std::ostream& s, const object& o) struct object_nil : object_class { bool isnil() const; bool operator== (const object_class* x) const; + void pack(dynamic_packer& p) const; const object_class* inspect(std::ostream& s) const; }; struct object_true : object_class { bool xbool() const; bool operator== (const object_class* x) const; + void pack(dynamic_packer& p) const; const object_class* inspect(std::ostream& s) const; }; struct object_false : object_class { bool xbool() const; bool operator== (const object_class* x) const; + void pack(dynamic_packer& p) const; const object_class* inspect(std::ostream& s) const; }; @@ -191,6 +197,7 @@ struct object_##NAME : object_class { \ bool operator== (const object_class* x) const; \ bool operator< (const object_class* x) const; \ bool operator> (const object_class* x) const; \ + void pack(dynamic_packer& p) const; \ const object_class* inspect(std::ostream& s) const; \ private: \ TYPE val; \ @@ -209,24 +216,25 @@ INTEGER_CLASS(int64_t, i64) #define FLOAT_CLASS(TYPE, NAME) \ -struct object_##NAME : object_class { \ - object_##NAME(TYPE v) : val(v) {} \ - uint8_t xu8 () const; \ - uint16_t xu16 () const; \ - uint32_t xu32 () const; \ - uint64_t xu64 () const; \ - int8_t xi8 () const; \ - int16_t xi16 () const; \ - int32_t xi32 () const; \ - int64_t xi64 () const; \ - float xfloat () const; \ - double xdouble() const; \ - bool operator== (const object_class* x) const; \ - bool operator< (const object_class* x) const; \ - bool operator> (const object_class* x) const; \ - const object_class* inspect(std::ostream& s) const; \ -private: \ - TYPE val; \ +struct object_##NAME : object_class { \ + object_##NAME(TYPE v) : val(v) {} \ + uint8_t xu8 () const; \ + uint16_t xu16 () const; \ + uint32_t xu32 () const; \ + uint64_t xu64 () const; \ + int8_t xi8 () const; \ + int16_t xi16 () const; \ + int32_t xi32 () const; \ + int64_t xi64 () const; \ + float xfloat () const; \ + double xdouble() const; \ + bool operator== (const object_class* x) const; \ + bool operator< (const object_class* x) const; \ + bool operator> (const object_class* x) const; \ + void pack(dynamic_packer& p) const; \ + const object_class* inspect(std::ostream& s) const; \ +private: \ + TYPE val; \ }; FLOAT_CLASS(float, float) @@ -242,6 +250,7 @@ struct object_##NAME : object_class { \ bool operator== (const object_class* x) const; \ bool operator< (const object_class* x) const; \ bool operator> (const object_class* x) const; \ + void pack(dynamic_packer& p) const; \ const object_class* inspect(std::ostream& s) const; \ private: \ TYPE ptr; \ @@ -261,6 +270,7 @@ struct object_array : object_class, object_container_mixin { const array& xarray() const; bool operator== (const object_class* x) const; // FIXME operator<, operator> + void pack(dynamic_packer& p) const; const object_class* inspect(std::ostream& s) const; public: void push_back(object o) { val.push_back(o); } @@ -276,6 +286,7 @@ struct object_map : object_class, object_container_mixin { const map& xmap() const; bool operator== (const object_class* x) const; // FIXME operator<, operator> + void pack(dynamic_packer& p) const; const object_class* inspect(std::ostream& s) const; public: void store(object k, object v) { val[k] = v; } diff --git a/cpp/pack.hpp b/cpp/pack.hpp new file mode 100644 index 0000000..89f8cee --- /dev/null +++ b/cpp/pack.hpp @@ -0,0 +1,192 @@ +#ifndef MSGPACK_PACK_HPP__ +#define MSGPACK_PACK_HPP__ + +#include "msgpack/object.hpp" +#include "msgpack/zone.hpp" +#include // __BYTE_ORDER +#include + +namespace msgpack { + + +template +class packer { +public: + packer(Stream& s); + +public: + void pack_int(int d) { pack_int_impl(m_stream, d); } + void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } + void pack_unsigned_int_8(uint8_t d) { pack_unsigned_int_8_impl(m_stream, d); } + void pack_unsigned_int_16(uint16_t d) { pack_unsigned_int_16_impl(m_stream, d); } + void pack_unsigned_int_32(uint32_t d) { pack_unsigned_int_32_impl(m_stream, d); } + void pack_unsigned_int_64(uint64_t d) { pack_unsigned_int_64_impl(m_stream, d); } + void pack_signed_int_8(uint8_t d) { pack_signed_int_8_impl(m_stream, d); } + void pack_signed_int_16(uint16_t d) { pack_signed_int_16_impl(m_stream, d); } + void pack_signed_int_32(uint32_t d) { pack_signed_int_32_impl(m_stream, d); } + void pack_signed_int_64(uint64_t d) { pack_signed_int_64_impl(m_stream, d); } + void pack_float(float d) { pack_float_impl(m_stream, d); } + void pack_double(double d) { pack_double_impl(m_stream, d); } + void pack_nil() { pack_nil(m_stream); } + void pack_true() { pack_true(m_stream); } + void pack_false() { pack_false(m_stream); } + void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } + void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } + void pack_string(const char* b) { pack_string_impl(m_stream, b); } + void pack_raw(const void* b, size_t l) { pack_raw_impl(m_stream, b, l); } + +private: + static void pack_int_impl(Stream& x, int d); + static void pack_unsigned_int_impl(Stream& x, unsigned int d); + static void pack_unsigned_int_8_impl(Stream& x, uint8_t d); + static void pack_unsigned_int_16_impl(Stream& x, uint16_t d); + static void pack_unsigned_int_32_impl(Stream& x, uint32_t d); + static void pack_unsigned_int_64_impl(Stream& x, uint64_t d); + static void pack_signed_int_8_impl(Stream& x, int8_t d); + static void pack_signed_int_16_impl(Stream& x, int16_t d); + static void pack_signed_int_32_impl(Stream& x, int32_t d); + static void pack_signed_int_64_impl(Stream& x, int64_t d); + static void pack_float_impl(Stream& x, float d); + static void pack_double_impl(Stream& x, double d); + static void pack_nil_impl(Stream& x); + static void pack_true_impl(Stream& x); + static void pack_false_impl(Stream& x); + static void pack_array_impl(Stream& x, unsigned int n); + static void pack_map_impl(Stream& x, unsigned int n); + static void pack_string_impl(Stream& x, const char* b); + static void pack_raw_impl(Stream& x, const void* b, size_t l); + static void append_buffer(Stream& x, const unsigned char* buf, unsigned int len) + { x.append((const char*)buf, len); } + +private: + Stream& m_stream; + +private: + packer(); +}; + +#define msgpack_pack_inline_func(name) \ + template \ + inline void packer::pack_ ## name ## _impl +#define msgpack_pack_context Stream& +#define msgpack_pack_append_buffer append_buffer +#include "msgpack/pack/inline_impl.h" +#undef msgpack_pack_context +#undef msgpack_pack_append_buffer + +template +packer::packer(Stream& s) : m_stream(s) { } + + +class dynamic_stream { +public: + template + dynamic_stream(Stream& s); +public: + dynamic_stream& append(const char* buf, size_t len) + { (*m_function)(m_object, buf, len); return *this; } +private: + void* m_object; + void (*m_function)(void* object, const char* buf, size_t len); +private: + template + static void append_trampoline(void* object, const char* buf, size_t len); +}; + + +class dynamic_packer { +public: + template + dynamic_packer(Stream& s); + +public: + void pack_int(int d) { pack_int_impl(m_stream, d); } + void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } + void pack_unsigned_int_8(uint8_t d) { pack_unsigned_int_8_impl(m_stream, d); } + void pack_unsigned_int_16(uint16_t d) { pack_unsigned_int_16_impl(m_stream, d); } + void pack_unsigned_int_32(uint32_t d) { pack_unsigned_int_32_impl(m_stream, d); } + void pack_unsigned_int_64(uint64_t d) { pack_unsigned_int_64_impl(m_stream, d); } + void pack_signed_int_8(uint8_t d) { pack_signed_int_8_impl(m_stream, d); } + void pack_signed_int_16(uint16_t d) { pack_signed_int_16_impl(m_stream, d); } + void pack_signed_int_32(uint32_t d) { pack_signed_int_32_impl(m_stream, d); } + void pack_signed_int_64(uint64_t d) { pack_signed_int_64_impl(m_stream, d); } + void pack_float(float d) { pack_float_impl(m_stream, d); } + void pack_double(double d) { pack_double_impl(m_stream, d); } + void pack_nil() { pack_nil_impl(m_stream); } + void pack_true() { pack_true_impl(m_stream); } + void pack_false() { pack_false_impl(m_stream); } + void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } + void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } + void pack_string(const char* b) { pack_string_impl(m_stream, b); } + void pack_raw(const void* b, size_t l) { pack_raw_impl(m_stream, b, l); } + +public: + +private: + static void pack_int_impl(dynamic_stream& x, int d); + static void pack_unsigned_int_impl(dynamic_stream& x, unsigned int d); + static void pack_unsigned_int_8_impl(dynamic_stream& x, uint8_t d); + static void pack_unsigned_int_16_impl(dynamic_stream& x, uint16_t d); + static void pack_unsigned_int_32_impl(dynamic_stream& x, uint32_t d); + static void pack_unsigned_int_64_impl(dynamic_stream& x, uint64_t d); + static void pack_signed_int_8_impl(dynamic_stream& x, int8_t d); + static void pack_signed_int_16_impl(dynamic_stream& x, int16_t d); + static void pack_signed_int_32_impl(dynamic_stream& x, int32_t d); + static void pack_signed_int_64_impl(dynamic_stream& x, int64_t d); + static void pack_float_impl(dynamic_stream& x, float d); + static void pack_double_impl(dynamic_stream& x, double d); + static void pack_nil_impl(dynamic_stream& x); + static void pack_true_impl(dynamic_stream& x); + static void pack_false_impl(dynamic_stream& x); + static void pack_array_impl(dynamic_stream& x, unsigned int n); + static void pack_map_impl(dynamic_stream& x, unsigned int n); + static void pack_string_impl(dynamic_stream& x, const char* b); + static void pack_raw_impl(dynamic_stream& x, const void* b, size_t l); + static void append_buffer(dynamic_stream& x, const unsigned char* buf, unsigned int len) + { x.append((const char*)buf, len); } + +private: + dynamic_stream m_stream; + +private: + dynamic_packer(); +}; + +#undef MSGPACK_PACK_INLINE_IMPL_H__ +#define msgpack_pack_inline_func(name) \ + inline void dynamic_packer::pack_ ## name ## _impl +#define msgpack_pack_context dynamic_stream& +#define msgpack_pack_append_buffer append_buffer +#include "msgpack/pack/inline_impl.h" +#undef msgpack_pack_context +#undef msgpack_pack_append_buffer + +template +dynamic_packer::dynamic_packer(Stream& s) : m_stream(s) { } + +template +dynamic_stream::dynamic_stream(Stream& s) +{ + m_object = reinterpret_cast(&s); + m_function = &dynamic_stream::append_trampoline; +} + +template +void dynamic_stream::append_trampoline(void* object, const char* buf, size_t len) +{ + (reinterpret_cast(object)->*MemFun)(buf, len); +} + + +template +inline void pack(Stream& s, object o) +{ + dynamic_packer pk(s); + o.pack(pk); +} + + +} // namespace msgpack + +#endif /* msgpack/pack.hpp */ + diff --git a/cpp/test.cpp b/cpp/test.cpp index 423a6bd..12e9150 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -1,12 +1,16 @@ #include +#include #include +#include class checker { public: void check(const char* d, size_t len, msgpack::object should) { + using msgpack::object; try { std::cout << "----" << std::endl; - msgpack::object o; + + object o; try { o = msgpack::unpack(d, len, m_zone); } catch (std::runtime_error& e) { @@ -14,10 +18,25 @@ public: std::cout << "**" << e.what() << "**" << std::endl; return; } + std::cout << o << std::endl; if(o != should) { std::cout << "** TEST FAILED **" << std::endl; } + + try { + std::string s; + msgpack::pack(s, o); + object ro = msgpack::unpack(s.data(), s.size(), m_zone); + if(ro != o) { throw std::runtime_error("NOT MATCH"); } + } catch (std::runtime_error& e) { + std::cout << "** REUNPACK FAILED **" << std::endl; + std::cout << e.what() << std::endl; + } catch (...) { + std::cout << "** REUNPACK FAILED **" << std::endl; + std::cout << "unknown error" << std::endl; + } + } catch (...) { m_zone.clear(); throw; } m_zone.clear(); } @@ -27,84 +46,82 @@ private: int main(void) { + checker c; -checker c; - -{ // SimpleValue - msgpack::zone z; - const char d[] = { - 0x93, 0xc0, 0xc2, 0xc3, - }; - c.check(d, sizeof(d), - z.narray( - z.nnil(), z.nfalse(), z.ntrue() - ) - ); -} - -{ // Fixnum - msgpack::zone z; - const char d[] = { - 0x92, - 0x93, 0x00, 0x40, 0x7f, - 0x93, 0xe0, 0xf0, 0xff, - }; - c.check(d, sizeof(d), - z.narray( + { // SimpleValue + msgpack::zone z; + const char d[] = { + 0x93, 0xc0, 0xc2, 0xc3, + }; + c.check(d, sizeof(d), z.narray( - z.nu8(0), - z.nu8(64), - z.nu8(127) - ), - z.narray( - z.ni8(-32), - z.ni8(-16), - z.ni8(-1) + z.nnil(), z.nfalse(), z.ntrue() ) - ) - ); -} + ); + } -{ // FixArray - msgpack::zone z; - const char d[] = { - 0x92, - 0x90, - 0x91, - 0x91, 0xc0, - }; - c.check(d, sizeof(d), - z.narray( - z.narray(), + { // Fixnum + msgpack::zone z; + const char d[] = { + 0x92, + 0x93, 0x00, 0x40, 0x7f, + 0x93, 0xe0, 0xf0, 0xff, + }; + c.check(d, sizeof(d), z.narray( z.narray( - z.nnil() + z.nu8(0), + z.nu8(64), + z.nu8(127) + ), + z.narray( + z.ni8(-32), + z.ni8(-16), + z.ni8(-1) ) ) - ) - ); -} - -{ // FixRaw - msgpack::zone z; - const char d[] = { - 0x94, - 0xa0, - 0xa1, 'a', - 0xa2, 'b', 'c', - 0xa3, 'd', 'e', 'f', - }; - c.check(d, sizeof(d), - z.narray( - z.nraw("", 0), - z.nraw("a", 1), - z.nraw("bc", 2), - z.nraw("def", 3) - ) - ); -} - - -return 0; + ); + } + + { // FixArray + msgpack::zone z; + const char d[] = { + 0x92, + 0x90, + 0x91, + 0x91, 0xc0, + }; + c.check(d, sizeof(d), + z.narray( + z.narray(), + z.narray( + z.narray( + z.nnil() + ) + ) + ) + ); + } + + { // FixRaw + msgpack::zone z; + const char d[] = { + 0x94, + 0xa0, + 0xa1, 'a', + 0xa2, 'b', 'c', + 0xa3, 'd', 'e', 'f', + }; + c.check(d, sizeof(d), + z.narray( + z.nraw("", 0), + z.nraw("a", 1), + z.nraw("bc", 2), + z.nraw("def", 3) + ) + ); + } + + return 0; } diff --git a/msgpack/pack/inline_impl.h b/msgpack/pack/inline_impl.h index 08a5bc1..d4d5a5a 100644 --- a/msgpack/pack/inline_impl.h +++ b/msgpack/pack/inline_impl.h @@ -18,8 +18,13 @@ #ifndef MSGPACK_PACK_INLINE_IMPL_H__ #define MSGPACK_PACK_INLINE_IMPL_H__ -#include -#include +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif #ifdef __LITTLE_ENDIAN__ @@ -47,13 +52,17 @@ #endif +#ifndef msgpack_pack_inline_func(name) +#define msgpack_pack_inline_func(name) \ + inline void msgpack_pack_##name +#endif /* * Integer */ // wrapper -inline void msgpack_pack_int(msgpack_pack_context x, int d) +msgpack_pack_inline_func(int)(msgpack_pack_context x, int d) { if(d < -32) { if(d < -32768) { // signed 32 @@ -86,11 +95,11 @@ inline void msgpack_pack_int(msgpack_pack_context x, int d) } // wrapper -inline void msgpack_pack_unsigned_int(msgpack_pack_context x, unsigned int d) +msgpack_pack_inline_func(unsigned_int)(msgpack_pack_context x, unsigned int d) { if(d < 128) { // fixnum - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); + msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); } else if(d < 256) { // unsigned 8 const unsigned char buf[2] = {0xcc, (uint8_t)d}; @@ -106,7 +115,7 @@ inline void msgpack_pack_unsigned_int(msgpack_pack_context x, unsigned int d) } } -inline void msgpack_pack_unsigned_int_8(msgpack_pack_context x, uint8_t d) +msgpack_pack_inline_func(unsigned_int_8)(msgpack_pack_context x, uint8_t d) { if(d < 128) { msgpack_pack_append_buffer(x, &d, 1); @@ -116,26 +125,26 @@ inline void msgpack_pack_unsigned_int_8(msgpack_pack_context x, uint8_t d) } } -inline void msgpack_pack_unsigned_int_16(msgpack_pack_context x, uint16_t d) +msgpack_pack_inline_func(unsigned_int_16)(msgpack_pack_context x, uint16_t d) { const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -inline void msgpack_pack_unsigned_int_32(msgpack_pack_context x, uint32_t d) +msgpack_pack_inline_func(unsigned_int_32)(msgpack_pack_context x, uint32_t d) { const unsigned char buf[5] = {0xce, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -inline void msgpack_pack_unsigned_int_64(msgpack_pack_context x, uint64_t d) +msgpack_pack_inline_func(unsigned_int_64)(msgpack_pack_context x, uint64_t d) { // FIXME optimization const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } -inline void msgpack_pack_signed_int_8(msgpack_pack_context x, int8_t d) +msgpack_pack_inline_func(signed_int_8)(msgpack_pack_context x, int8_t d) { if(d > 0) { msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); @@ -147,19 +156,19 @@ inline void msgpack_pack_signed_int_8(msgpack_pack_context x, int8_t d) } } -inline void msgpack_pack_signed_int_16(msgpack_pack_context x, int16_t d) +msgpack_pack_inline_func(signed_int_16)(msgpack_pack_context x, int16_t d) { const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -inline void msgpack_pack_signed_int_32(msgpack_pack_context x, int32_t d) +msgpack_pack_inline_func(signed_int_32)(msgpack_pack_context x, int32_t d) { const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -inline void msgpack_pack_signed_int_64(msgpack_pack_context x, int64_t d) +msgpack_pack_inline_func(signed_int_64)(msgpack_pack_context x, int64_t d) { // FIXME optimization const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; @@ -171,14 +180,14 @@ inline void msgpack_pack_signed_int_64(msgpack_pack_context x, int64_t d) * Float */ -inline void msgpack_pack_float(msgpack_pack_context x, float d) +msgpack_pack_inline_func(float)(msgpack_pack_context x, float d) { uint32_t n = *((uint32_t*)&d); // FIXME const unsigned char buf[5] = {0xca, STORE_BE32(n)}; msgpack_pack_append_buffer(x, buf, 5); } -inline void msgpack_pack_double(msgpack_pack_context x, double d) +msgpack_pack_inline_func(double)(msgpack_pack_context x, double d) { uint64_t n = *((uint64_t*)&d); // FIXME const unsigned char buf[9] = {0xcb, STORE_BE64(n)}; @@ -190,7 +199,7 @@ inline void msgpack_pack_double(msgpack_pack_context x, double d) * Nil */ -inline void msgpack_pack_nil(msgpack_pack_context x) +msgpack_pack_inline_func(nil)(msgpack_pack_context x) { static const unsigned char d = 0xc0; msgpack_pack_append_buffer(x, &d, 1); @@ -201,13 +210,13 @@ inline void msgpack_pack_nil(msgpack_pack_context x) * Boolean */ -inline void msgpack_pack_true(msgpack_pack_context x) +msgpack_pack_inline_func(true)(msgpack_pack_context x) { static const unsigned char d = 0xc3; msgpack_pack_append_buffer(x, &d, 1); } -inline void msgpack_pack_false(msgpack_pack_context x) +msgpack_pack_inline_func(false)(msgpack_pack_context x) { static const unsigned char d = 0xc2; msgpack_pack_append_buffer(x, &d, 1); @@ -218,7 +227,7 @@ inline void msgpack_pack_false(msgpack_pack_context x) * Array */ -inline void msgpack_pack_array(msgpack_pack_context x, unsigned int n) +msgpack_pack_inline_func(array)(msgpack_pack_context x, unsigned int n) { if(n < 16) { unsigned char d = 0x90 | n; @@ -239,7 +248,7 @@ inline void msgpack_pack_array(msgpack_pack_context x, unsigned int n) * Map */ -inline void msgpack_pack_map(msgpack_pack_context x, unsigned int n) +msgpack_pack_inline_func(map)(msgpack_pack_context x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; @@ -260,13 +269,13 @@ inline void msgpack_pack_map(msgpack_pack_context x, unsigned int n) * Raw */ -inline void msgpack_pack_string(msgpack_pack_context x, const char* b) +msgpack_pack_inline_func(string)(msgpack_pack_context x, const char* b) { uint32_t l = strlen(b); msgpack_pack_append_buffer(x, (const unsigned char*)b, l+1); } -inline void msgpack_pack_raw(msgpack_pack_context x, const void* b, size_t l) +msgpack_pack_inline_func(raw)(msgpack_pack_context x, const void* b, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; @@ -280,14 +289,15 @@ inline void msgpack_pack_raw(msgpack_pack_context x, const void* b, size_t l) unsigned char buf[5] = {0xdb, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } - msgpack_pack_append_buffer(x, b, l); + msgpack_pack_append_buffer(x, (const unsigned char*)b, l); } +#undef msgpack_pack_inline_func(name) + #undef STORE_BE16(d) #undef STORE_BE32(d) #undef STORE_BE64(d) - #endif /* msgpack/pack/inline_impl.h */ diff --git a/ruby/pack_inline.h b/ruby/pack_inline.h index ab4b092..bda74c3 100644 --- a/ruby/pack_inline.h +++ b/ruby/pack_inline.h @@ -27,6 +27,8 @@ static inline void msgpack_pack_append_buffer(VALUE x, const unsigned char* b, u rb_str_buf_cat(x, (const void*)b, l); } +#include +#include /* __BYTE_ORDER */ #include "msgpack/pack/inline_impl.h" #endif /* pack_inline.h */ From 249d3e9c908ea4a3fd2012a753ca7f0cea0b4ee5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0007/1172] lang/c/msgpack: removed string type git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@54 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/bench.c | 6 ++---- c/bench_inline.c | 4 +--- c/unpack.h | 3 +-- c/unpack_inline.c | 7 ++----- msgpack/unpack/callback.h | 2 +- msgpack/unpack/inline_context.h | 2 +- msgpack/unpack/inline_impl.h | 36 +++++++++++++++++---------------- ruby/test_format.rb | 6 ------ ruby/unpack_inline.c | 7 ++----- 9 files changed, 29 insertions(+), 44 deletions(-) diff --git a/c/bench.c b/c/bench.c index f27350a..91f0c21 100644 --- a/c/bench.c +++ b/c/bench.c @@ -55,8 +55,7 @@ static void* unpack_array_start(void* data, unsigned int n) { return NULL; } static void unpack_array_item(void* data, void* c, void* o) { } static void* unpack_map_start(void* data, unsigned int n) { return NULL; } static void unpack_map_item(void* data, void* c, void* k, void* v) { } -static void* unpack_string(void* data, const void* b, size_t l) { return NULL; } -static void* unpack_raw(void* data, const void* b, size_t l) { /*printf("unpack raw %p %lu\n",b,l);*/ return NULL; } +static void* unpack_raw(void* data, const void* b, const void* p, size_t l) { /*printf("unpack raw %p %lu\n",p,l);*/ return NULL; } typedef struct { size_t allocated; @@ -119,7 +118,7 @@ void bench_json(void) NULL, NULL, reformat_number, - reformat_string, + reformat_string, reformat_start_map, reformat_map_key, reformat_end_map, @@ -236,7 +235,6 @@ void bench_msgpack(void) unpack_array_item, unpack_map_start, unpack_map_item, - unpack_string, unpack_raw, }; msgpack_unpack_t* mupk = msgpack_unpack_new(NULL, &cb); diff --git a/c/bench_inline.c b/c/bench_inline.c index 2901508..5390319 100644 --- a/c/bench_inline.c +++ b/c/bench_inline.c @@ -65,8 +65,7 @@ static inline void* msgpack_unpack_array_start(msgpack_unpack_context* x, unsign static inline void msgpack_unpack_array_item(msgpack_unpack_context* x, void* c, void* o) { } static inline void* msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n) { return NULL; } static inline void msgpack_unpack_map_item(msgpack_unpack_context* x, void* c, void* k, void* v) { } -static inline void* msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l) { return NULL; } -static inline void* msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, size_t l) { return NULL; } +static inline void* msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, const void* p, size_t l) { return NULL; } #include "msgpack/unpack/inline_impl.h" @@ -131,7 +130,6 @@ void bench_json(void) NULL, NULL, reformat_number, - reformat_string, reformat_start_map, reformat_map_key, reformat_end_map, diff --git a/c/unpack.h b/c/unpack.h index 3230a0d..094328e 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -39,8 +39,7 @@ typedef struct { void (*unpack_array_item)(void* data, void* c, void* o); void* (*unpack_map_start)(void* data, unsigned int n); void (*unpack_map_item)(void* data, void* c, void* k, void* v); - void* (*unpack_string)(void* data, const void* b, size_t l); - void* (*unpack_raw)(void* data, const void* b, size_t l); + void* (*unpack_raw)(void* data, const void* b, const void* p, size_t l); } msgpack_unpack_callback; typedef struct { diff --git a/c/unpack_inline.c b/c/unpack_inline.c index 3525468..d758d3e 100644 --- a/c/unpack_inline.c +++ b/c/unpack_inline.c @@ -71,11 +71,8 @@ static inline void* msgpack_unpack_map_start(msgpack_unpack_t* x, unsigned int n static inline void msgpack_unpack_map_item(msgpack_unpack_t* x, void* c, void* k, void* v) { x->callback.unpack_map_item(x->data, c, k, v); } -static inline void* msgpack_unpack_string(msgpack_unpack_t* x, const void* b, size_t l) -{ return x->callback.unpack_string(x->data, b, l); } - -static inline void* msgpack_unpack_raw(msgpack_unpack_t* x, const void* b, size_t l) -{ return x->callback.unpack_raw(x->data, b, l); } +static inline void* msgpack_unpack_raw(msgpack_unpack_t* x, const void* b, const void* p, size_t l) +{ return x->callback.unpack_raw(x->data, b, p, l); } #include "msgpack/unpack/inline_impl.h" diff --git a/msgpack/unpack/callback.h b/msgpack/unpack/callback.h index b058a15..51508cc 100644 --- a/msgpack/unpack/callback.h +++ b/msgpack/unpack/callback.h @@ -20,5 +20,5 @@ msgpack_object msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned in msgpack_object msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n); void msgpack_unpack_map_item(msgpack_unpack_context* x, msgpack_object c, msgpack_object k, msgpack_object v); msgpack_object msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l); -msgpack_object msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, size_t l); +msgpack_object msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, const void* p, size_t l); diff --git a/msgpack/unpack/inline_context.h b/msgpack/unpack/inline_context.h index d6558a3..22698d5 100644 --- a/msgpack/unpack/inline_context.h +++ b/msgpack/unpack/inline_context.h @@ -34,7 +34,7 @@ typedef struct { size_t count; unsigned int ct; union { - const unsigned char* terminal_trail_start; + /*const unsigned char* terminal_trail_start;*/ msgpack_object map_key; } tmp; } msgpack_unpacker_stack; diff --git a/msgpack/unpack/inline_impl.h b/msgpack/unpack/inline_impl.h index f610dd7..fdbe7f6 100644 --- a/msgpack/unpack/inline_impl.h +++ b/msgpack/unpack/inline_impl.h @@ -103,7 +103,7 @@ static inline uint64_t betoh64(uint64_t x) { typedef enum { CS_HEADER = 0x00, // nil - CS_STRING = 0x01, + //CS_STRING = 0x01, //CS_ = 0x02, // false //CS_ = 0x03, // true @@ -186,15 +186,17 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len obj = func(user, arg); \ /*printf("obj %d\n",obj);*/ \ goto _push -#define push_variable_value(func, arg, arglen) \ - obj = func(user, arg, arglen); \ +#define push_variable_value(func, base, pos, len) \ + obj = func(user, (const void*)base, (const void*)pos, len); \ /*printf("obj %d\n",obj);*/ \ goto _push +/* #define again_terminal_trail(_cs, from) \ cs = _cs; \ stack[top].tmp.terminal_trail_start = from; \ goto _terminal_trail_again +*/ #define again_fixed_trail(_cs, trail_len) \ trail = trail_len; \ cs = _cs; \ @@ -237,8 +239,8 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len switch(*p) { case 0xc0: // nil push_simple_value(msgpack_unpack_nil); - case 0xc1: // string - again_terminal_trail(NEXT_CS(p), p+1); + //case 0xc1: // string + // again_terminal_trail(NEXT_CS(p), p+1); case 0xc2: // false push_simple_value(msgpack_unpack_false); case 0xc3: // true @@ -289,16 +291,16 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len // end CS_HEADER - _terminal_trail_again: - ++p; + //_terminal_trail_again: + // ++p; - case CS_STRING: - if(*p == 0) { - const unsigned char* start = stack[top].tmp.terminal_trail_start; - obj = msgpack_unpack_string(user, start, p-start); - goto _push; - } - goto _terminal_trail_again; + //case CS_STRING: + // if(*p == 0) { + // const unsigned char* start = stack[top].tmp.terminal_trail_start; + // obj = msgpack_unpack_string(user, start, p-start); + // goto _push; + // } + // goto _terminal_trail_again; _fixed_trail_again: @@ -343,7 +345,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len //case ACS_BIG_INT_VALUE: //_big_int_zero: // // FIXME - // push_variable_value(msgpack_unpack_big_int, n, trail); + // push_variable_value(msgpack_unpack_big_int, data, n, trail); //case CS_BIG_FLOAT_16: // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); @@ -352,7 +354,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len //case ACS_BIG_FLOAT_VALUE: //_big_float_zero: // // FIXME - // push_variable_value(msgpack_unpack_big_float, n, trail); + // push_variable_value(msgpack_unpack_big_float, data, n, trail); case CS_RAW_16: again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); @@ -360,7 +362,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); case ACS_RAW_VALUE: _raw_zero: - push_variable_value(msgpack_unpack_raw, n, trail); + push_variable_value(msgpack_unpack_raw, data, n, trail); case CS_ARRAY_16: start_container(msgpack_unpack_array_start, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); diff --git a/ruby/test_format.rb b/ruby/test_format.rb index 7c2e8fc..99a27d1 100644 --- a/ruby/test_format.rb +++ b/ruby/test_format.rb @@ -119,10 +119,4 @@ check([ 0xdf, 0x00, 0x00, 0x00, 0x02, 0xc0, 0xc2, 0xc3, 0xc2, ], [{}, {nil=>false}, {true=>false, nil=>false}, {}, {nil=>false}, {true=>false, nil=>false}]) -# string -check([ - 0x92, - 0xc1, 0x00, - 0xc1, ?a, ?b, ?c, 0x00, -], ["", "abc"]) diff --git a/ruby/unpack_inline.c b/ruby/unpack_inline.c index f6715d9..b30754b 100644 --- a/ruby/unpack_inline.c +++ b/ruby/unpack_inline.c @@ -71,11 +71,8 @@ static inline VALUE msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned static inline void msgpack_unpack_map_item(msgpack_unpack_context* x, VALUE c, VALUE k, VALUE v) { rb_hash_aset(c, k, v); } -static inline VALUE msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l) -{ return rb_str_new(b, l); } - -static inline VALUE msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, size_t l) -{ return rb_str_new(b, l); } +static inline VALUE msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, const void* p, size_t l) +{ return rb_str_new(p, l); } #include "msgpack/unpack/inline_impl.h" From 990ac38ccdb51acc520fa43c99115cb216ec95e6 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0008/1172] lang/c/msgpack: C++ binding: implemented built-in buffer. git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@55 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/Makefile | 3 +- cpp/object.cpp | 10 ++-- cpp/object.hpp | 9 ++-- cpp/test.cpp | 81 +++++++++++++++++++++++++++-- cpp/unpack.cpp | 115 ++++++++++++++++++++++++++++++++++-------- cpp/unpack.hpp | 48 +++++++++++++++--- cpp/unpack_inline.cpp | 7 +-- cpp/zone.cpp | 29 +++++++---- cpp/zone.hpp.erb | 63 +++++++++++++++++++---- 9 files changed, 299 insertions(+), 66 deletions(-) diff --git a/cpp/Makefile b/cpp/Makefile index 5ef7ecd..7cc66b2 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -1,5 +1,6 @@ -CXXFLAGS = -I.. -I. -Wall -g -O4 +CXXFLAGS = -I.. -I. -Wall -g +#CXXFLAGS = -I.. -I. -Wall -g -O4 LDFLAGS = -L. NEED_PREPROCESS = zone.hpp diff --git a/cpp/object.cpp b/cpp/object.cpp index 02e0438..947e8dd 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -320,12 +320,12 @@ const object_class* object_##NAME::inspect(std::ostream& s) const \ { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape -RAW_OBJECT(raw, - raw object_raw::xraw() { return raw(ptr, len); } - const_raw object_raw::xraw() const { return const_raw(ptr, len); } ) +RAW_OBJECT(raw_ref, + raw object_raw_ref::xraw() { return raw(ptr, len); } + const_raw object_raw_ref::xraw() const { return const_raw(ptr, len); } ) -RAW_OBJECT(const_raw, - const_raw object_const_raw::xraw() const { return const_raw(ptr, len); } ) +RAW_OBJECT(const_raw_ref, + const_raw object_const_raw_ref::xraw() const { return const_raw(ptr, len); } ) #undef RAW_OBJECT(NAME, EXTRA) diff --git a/cpp/object.hpp b/cpp/object.hpp index 3f22dfc..f029e32 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -31,8 +31,9 @@ public: }; struct const_raw { - const_raw() : ptr(NULL), len(0) {} - const_raw(const void* p, size_t l) : ptr(p), len(l) {} + explicit const_raw() : ptr(NULL), len(0) {} + explicit const_raw(const void* p, size_t l) : ptr(p), len(l) {} + const_raw(const raw& m) : ptr(m.ptr), len(m.len) {} public: const void* ptr; size_t len; @@ -257,8 +258,8 @@ private: \ uint32_t len; \ }; -RAW_CLASS(raw, void*, raw xraw(); const_raw xraw() const; ) -RAW_CLASS(const_raw, const void*, const_raw xraw() const; ) +RAW_CLASS(raw_ref, void*, raw xraw(); const_raw xraw() const; ) +RAW_CLASS(const_raw_ref, const void*, const_raw xraw() const; ) #undef RAW_CLASS(NAME, TYPE, EXTRA) diff --git a/cpp/test.cpp b/cpp/test.cpp index 12e9150..3756a05 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include class checker { public: @@ -114,14 +116,85 @@ int main(void) }; c.check(d, sizeof(d), z.narray( - z.nraw("", 0), - z.nraw("a", 1), - z.nraw("bc", 2), - z.nraw("def", 3) + z.nraw_ref("", 0), + z.nraw_ref("a", 1), + z.nraw_ref("bc", 2), + z.nraw_ref("def", 3) ) ); } + static const uint16_t TASK_ARRAY = 100; + static char tarray[3]; + static char traw[64]; + + { + memset(traw, 'a', sizeof(traw)); + traw[0] = 0xda; + uint16_t n = htons(sizeof(traw)-3); + traw[1] = ((char*)&n)[0]; + traw[2] = ((char*)&n)[1]; + + msgpack::zone z; + std::cout << msgpack::unpack(traw, sizeof(traw), z) << std::endl;; + } + + { + tarray[0] = 0xdc; + uint16_t n = htons(TASK_ARRAY); + tarray[1] = ((char*)&n)[0]; + tarray[2] = ((char*)&n)[1]; + } + + { + // write message + ssize_t total_bytes = 0; + std::stringstream stream; + for(unsigned q=0; q < 10; ++q) { + stream.write(tarray, sizeof(tarray)); + total_bytes += sizeof(tarray); + for(uint16_t i=0; i < TASK_ARRAY; ++i) { + stream.write(traw, sizeof(traw)); + total_bytes += sizeof(traw); + } + } + + stream.seekg(0); + + // reserive message + unsigned num_msg = 0; + + static const size_t RESERVE_SIZE = 32;//*1024; + + msgpack::unpacker upk; + while(stream.good() && total_bytes > 0) { + + upk.reserve_buffer(RESERVE_SIZE); + size_t sz = stream.readsome( + (char*)upk.buffer(), + upk.buffer_capacity()); + + total_bytes -= sz; + std::cout << "read " << sz << " bytes to capacity " + << upk.buffer_capacity() << " bytes" + << std::endl; + + upk.buffer_consumed(sz); + while( upk.execute() ) { + std::cout << "message parsed" << std::endl; + boost::scoped_ptr pz(upk.release_zone()); + msgpack::object o = upk.data(); + upk.reset(); + std::cout << o << std::endl; + ++num_msg; + } + + } + + std::cout << "stream finished" << std::endl; + std::cout << num_msg << " messages reached" << std::endl; + } + return 0; } diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 0f02d3c..5055008 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -5,10 +5,10 @@ namespace msgpack { struct unpacker::context { - context(zone& z) + context(zone* z) { msgpack_unpacker_init(&m_ctx); - m_ctx.user = &z; + m_ctx.user = z; } ~context() { } @@ -30,6 +30,22 @@ struct unpacker::context { m_ctx.user = z; } + void reset(zone* z) + { + msgpack_unpacker_init(&m_ctx); + m_ctx.user = z; + } + + zone* user() + { + return m_ctx.user; + } + + void user(zone* z) + { + m_ctx.user = z; + } + private: msgpack_unpacker m_ctx; @@ -39,46 +55,105 @@ private: }; -unpacker::unpacker(zone& z) : - m_ctx(new context(z)), - m_zone(z), - m_finished(false) +unpacker::unpacker() : + m_zone(new zone()), + m_ctx(new context(m_zone)), + m_buffer(NULL), + m_used(0), + m_free(0), + m_off(0) { } -unpacker::~unpacker() { delete m_ctx; } - - -size_t unpacker::execute(const void* data, size_t len, size_t off) +unpacker::~unpacker() { - int ret = m_ctx->execute(data, len, &off); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret > 0) { - m_finished = true; - return off; + free(m_buffer); + delete m_ctx; + delete m_zone; +} + + +void unpacker::expand_buffer(size_t len) +{ + if(m_off == 0) { + size_t next_size; + if(m_free != 0) { next_size = m_free * 2; } + else { next_size = MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE; } + while(next_size < len + m_used) { next_size *= 2; } + + // FIXME realloc? + + void* tmp = malloc(next_size); + if(!tmp) { throw std::bad_alloc(); } + memcpy(tmp, m_buffer, m_used); + + free(m_buffer); + m_buffer = tmp; + m_free = next_size - m_used; + } else { - m_finished = false; - return off; + size_t next_size = MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE; + while(next_size < len + m_used - m_off) { next_size *= 2; } + + void* tmp = malloc(next_size); + if(!tmp) { throw std::bad_alloc(); } + memcpy(tmp, ((char*)m_buffer)+m_off, m_used-m_off); + + try { + m_zone->push_finalizer(&zone::finalize_free, NULL, m_buffer); + } catch (...) { + free(tmp); + throw; + } + + m_buffer = tmp; + m_used = m_used - m_off; + m_free = next_size - m_used; + m_off = 0; } } +bool unpacker::execute() +{ + int ret = m_ctx->execute(m_buffer, m_used, &m_off); + if(ret < 0) { + throw unpack_error("parse error"); + } else if(ret == 0) { + return false; + } else { + return true; + } +} + +zone* unpacker::release_zone() +{ + zone* z = m_zone; + m_zone = NULL; + m_zone = new zone(); + m_ctx->user(m_zone); + return z; +} object unpacker::data() { return object(m_ctx->data()); } - void unpacker::reset() { + if(!m_zone->empty()) { + delete m_zone; + m_zone = NULL; + m_zone = new zone(); + } + expand_buffer(0); m_ctx->reset(); } object unpacker::unpack(const void* data, size_t len, zone& z) { - context ctx(z); + context ctx(&z); size_t off = 0; int ret = ctx.execute(data, len, &off); if(ret < 0) { diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 61ba781..df4a636 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -5,6 +5,10 @@ #include "msgpack/zone.hpp" #include +#ifndef MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE +#define MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE 8*1024 +#endif + namespace msgpack { @@ -16,26 +20,58 @@ struct unpack_error : public std::runtime_error { class unpacker { public: - unpacker(zone& z); + unpacker(); ~unpacker(); + public: - size_t execute(const void* data, size_t len, size_t off); - bool is_finished() { return m_finished; } + void reserve_buffer(size_t len); + void* buffer(); + size_t buffer_capacity() const; + void buffer_consumed(size_t len); + bool execute(); + zone* release_zone(); // never throw object data(); void reset(); + private: + zone* m_zone; + struct context; context* m_ctx; - zone& m_zone; - bool m_finished; + + void* m_buffer; + size_t m_used; + size_t m_free; + size_t m_off; + void expand_buffer(size_t len); + private: - unpacker(); unpacker(const unpacker&); + public: static object unpack(const void* data, size_t len, zone& z); }; +inline void unpacker::reserve_buffer(size_t len) +{ + if(m_free >= len) { return; } + expand_buffer(len); +} + +inline void* unpacker::buffer() + { return (void*)(((char*)m_buffer)+m_used); } + +inline size_t unpacker::buffer_capacity() const + { return m_free; } + +inline void unpacker::buffer_consumed(size_t len) +{ + m_used += len; + m_free -= len; +} + + inline object unpack(const void* data, size_t len, zone& z) { return unpacker::unpack(data, len, z); diff --git a/cpp/unpack_inline.cpp b/cpp/unpack_inline.cpp index 82f6e7a..9002327 100644 --- a/cpp/unpack_inline.cpp +++ b/cpp/unpack_inline.cpp @@ -59,11 +59,8 @@ static inline object_class* msgpack_unpack_map_start(zone** z, unsigned int n) static inline void msgpack_unpack_map_item(zone** z, object_class* c, object_class* k, object_class* v) { reinterpret_cast(c)->store(k, v); } -static inline object_class* msgpack_unpack_string(zone** z, const void* b, size_t l) -{ return (*z)->nraw(b, l); } - -static inline object_class* msgpack_unpack_raw(zone** z, const void* b, size_t l) -{ return (*z)->nraw(b, l); } +static inline object_class* msgpack_unpack_raw(zone** z, const void* b, const void* p, size_t l) +{ return (*z)->nraw_ref(p, l); } } // extern "C" diff --git a/cpp/zone.cpp b/cpp/zone.cpp index 5031467..de2de22 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -6,27 +6,36 @@ namespace msgpack { void* zone::alloc() { if(m_used >= m_pool.size()*MSGPACK_ZONE_CHUNK_SIZE) { - m_pool.push_back(chunk_t()); + m_pool.push_back(new chunk_t()); } - void* data = m_pool[m_used/MSGPACK_ZONE_CHUNK_SIZE].cells[m_used%MSGPACK_ZONE_CHUNK_SIZE].data; + void* data = m_pool[m_used/MSGPACK_ZONE_CHUNK_SIZE]->cells[m_used%MSGPACK_ZONE_CHUNK_SIZE].data; ++m_used; return data; } void zone::clear() { - for(size_t b=0; b < m_used/MSGPACK_ZONE_CHUNK_SIZE; ++b) { - cell_t* c(m_pool[b].cells); - for(size_t e=0; e < MSGPACK_ZONE_CHUNK_SIZE; ++e) { + if(!m_pool.empty()) { + for(size_t b=0; b < m_used/MSGPACK_ZONE_CHUNK_SIZE; ++b) { + cell_t* c(m_pool[b]->cells); + for(size_t e=0; e < MSGPACK_ZONE_CHUNK_SIZE; ++e) { + reinterpret_cast(c[e].data)->~object_class(); + } + } + cell_t* c(m_pool.back()->cells); + for(size_t e=0; e < m_used%MSGPACK_ZONE_CHUNK_SIZE; ++e) { reinterpret_cast(c[e].data)->~object_class(); } - } - cell_t* c(m_pool.back().cells); - for(size_t e=0; e < m_used%MSGPACK_ZONE_CHUNK_SIZE; ++e) { - reinterpret_cast(c[e].data)->~object_class(); + + for(pool_t::iterator it(m_pool.begin()), it_end(m_pool.end()); + it != it_end; + ++it) { + delete *it; + } + m_pool.clear(); } m_used = 0; - m_pool.resize(1); + for(user_finalizer_t::reverse_iterator it(m_user_finalizer.rbegin()), it_end(m_user_finalizer.rend()); it != it_end; ++it) { diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 1a941af..40ce694 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -1,8 +1,11 @@ #ifndef MSGPACK_ZONE_HPP__ #define MSGPACK_ZONE_HPP__ +#include #include "msgpack/object.hpp" -#include +#include +#include +#include #ifndef MSGPACK_ZONE_CHUNK_SIZE #define MSGPACK_ZONE_CHUNK_SIZE 64 @@ -13,8 +16,8 @@ namespace msgpack { class zone { public: -zone() : m_used(0), m_pool(1) { } -~zone() { clear(); } + zone() : m_used(0) { } + ~zone() { clear(); } public: template @@ -35,11 +38,27 @@ public: object_float* nfloat( float v) { return new (alloc()) object_float(v); } object_double* ndouble( double v) { return new (alloc()) object_double(v); } - object_raw* nraw(void* ptr, uint32_t len) - { return new (alloc()) object_raw(ptr, len); } + object_raw_ref* nraw_ref(void* ptr, uint32_t len) + { return new (alloc()) object_raw_ref(ptr, len); } - object_const_raw* nraw(const void* ptr, uint32_t len) - { return new (alloc()) object_const_raw(ptr, len); } + object_const_raw_ref* nraw_ref(const void* ptr, uint32_t len) + { return new (alloc()) object_const_raw_ref(ptr, len); } + + object_raw_ref* nraw_copy(const void* ptr, uint32_t len) + { + void* copy = malloc(len); + if(!copy) { throw std::bad_alloc(); } + object_raw_ref* o; + try { + o = new (alloc()) object_raw_ref(copy, len); + push_finalizer(&zone::finalize_free, NULL, copy); + } catch (...) { + free(copy); + throw; + } + memcpy(copy, ptr, len); + return o; + } object_array* narray() { return new (alloc()) object_array(); } @@ -67,6 +86,7 @@ public: public: void clear(); + bool empty() const; private: void* alloc(); @@ -75,9 +95,9 @@ private: size_t m_used; static const size_t MAX_OBJECT_SIZE = - sizeof(object_raw) > sizeof(object_array) - ? ( sizeof(object_raw) > sizeof(object_map) - ? sizeof(object_raw) + sizeof(object_raw_ref) > sizeof(object_array) + ? ( sizeof(object_raw_ref) > sizeof(object_map) + ? sizeof(object_raw_ref) : sizeof(object_map) ) : ( sizeof(object_array) > sizeof(object_map) @@ -94,7 +114,7 @@ private: cell_t cells[MSGPACK_ZONE_CHUNK_SIZE]; }; - typedef std::vector pool_t; + typedef std::vector pool_t; pool_t m_pool; @@ -112,11 +132,32 @@ private: typedef std::vector user_finalizer_t; user_finalizer_t m_user_finalizer; +private: + void resize_pool(size_t n); + +public: + static void finalize_free(void* obj, void* user) + { free(user); } + private: zone(const zone&); }; +template +inline void zone::push_finalizer(void (*func)(void* obj, void* user), T* obj, void* user) +{ + m_user_finalizer.push_back( finalizer( + func, reinterpret_cast(obj), + user) ); +} + +inline bool zone::empty() const +{ + return m_used == 0 && m_user_finalizer.empty(); +} + + } // namespace msgpack #endif /* msgpack/zone.hpp */ From 7e6b55a71805dabb2ad82bf6802ea81f36bc911a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0009/1172] lang/c/msgpack: C++ binding: support non-MessagePack message that follows after MessagePack message git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@56 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/test.cpp | 17 +++++++++++++++-- cpp/unpack.cpp | 8 ++++---- cpp/unpack.hpp | 46 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/cpp/test.cpp b/cpp/test.cpp index 3756a05..e236d0f 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -169,7 +169,10 @@ int main(void) msgpack::unpacker upk; while(stream.good() && total_bytes > 0) { + // 1. reserve buffer upk.reserve_buffer(RESERVE_SIZE); + + // 2. read data to buffer() up to buffer_capacity() bytes size_t sz = stream.readsome( (char*)upk.buffer(), upk.buffer_capacity()); @@ -179,14 +182,24 @@ int main(void) << upk.buffer_capacity() << " bytes" << std::endl; + // 3. specify the number of bytes actually copied upk.buffer_consumed(sz); + + // 4. repeat execute() until it returns false while( upk.execute() ) { std::cout << "message parsed" << std::endl; - boost::scoped_ptr pz(upk.release_zone()); + + // 5.1. take out the parsed object msgpack::object o = upk.data(); - upk.reset(); + + // 5.2. the parsed object is valid until the zone is deleted + boost::scoped_ptr pz(upk.release_zone()); + std::cout << o << std::endl; ++num_msg; + + // 5.3 re-initialize unpacker + upk.reset(); } } diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 5055008..d484593 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -82,7 +82,6 @@ void unpacker::expand_buffer(size_t len) while(next_size < len + m_used) { next_size *= 2; } // FIXME realloc? - void* tmp = malloc(next_size); if(!tmp) { throw std::bad_alloc(); } memcpy(tmp, m_buffer, m_used); @@ -121,15 +120,16 @@ bool unpacker::execute() } else if(ret == 0) { return false; } else { + expand_buffer(0); return true; } } zone* unpacker::release_zone() { + zone* nz = new zone(); zone* z = m_zone; - m_zone = NULL; - m_zone = new zone(); + m_zone = nz; m_ctx->user(m_zone); return z; } @@ -141,12 +141,12 @@ object unpacker::data() void unpacker::reset() { + if(m_off != 0) { expand_buffer(0); } if(!m_zone->empty()) { delete m_zone; m_zone = NULL; m_zone = new zone(); } - expand_buffer(0); m_ctx->reset(); } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index df4a636..ae536c9 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -24,15 +24,46 @@ public: ~unpacker(); public: + /*! 1. reserve buffer. at least `len' bytes of capacity will be ready */ void reserve_buffer(size_t len); + + /*! 2. read data to the buffer() up to buffer_capacity() bytes */ void* buffer(); size_t buffer_capacity() const; + + /*! 3. specify the number of bytes actually copied */ void buffer_consumed(size_t len); + + /*! 4. repeat execute() until it retunrs false */ bool execute(); - zone* release_zone(); // never throw + + /*! 5.1. if execute() returns true, take out the parsed object */ object data(); + + /*! 5.2. the parsed object is valid until the zone is deleted */ + // Note that once release_zone() from unpacker, you must delete it + // otherwise the memrory will leak. + zone* release_zone(); + + /*! 5.3. after release_zone(), re-initialize unpacker */ void reset(); +public: + // These functions are usable when non-MessagePack message follows after + // MessagePack message. + // Note that there are no parsed buffer when execute() returned true. + + /*! get address of buffer that is not parsed */ + void* nonparsed_buffer(); + size_t nonparsed_size() const; + + /*! get the number of bytes that is already parsed */ + size_t parsed_size() const; + + /*! remove unparsed buffer from unpacker */ + // Note that reset() leaves non-parsed buffer. + void remove_nonparsed_buffer(); + private: zone* m_zone; @@ -72,6 +103,19 @@ inline void unpacker::buffer_consumed(size_t len) } +inline void* unpacker::nonparsed_buffer() + { return (void*)(((char*)m_buffer)+m_off); } + +inline size_t unpacker::nonparsed_size() const + { return m_used - m_off; } + +inline size_t unpacker::parsed_size() const + { return m_off; } + +inline void unpacker::remove_nonparsed_buffer() + { m_used = m_off; } + + inline object unpack(const void* data, size_t len, zone& z) { return unpacker::unpack(data, len, z); From 94cbe54cf377f16bcc18b80d8b6f81eefd50e723 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0010/1172] lang/c/msgpack: C++ binding: added bench.cpp git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@57 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/Makefile | 15 ++-- cpp/bench.cpp | 181 +++++++++++++++++++++++++++++++++++++++++++++++++ cpp/unpack.cpp | 2 +- 3 files changed, 191 insertions(+), 7 deletions(-) create mode 100644 cpp/bench.cpp diff --git a/cpp/Makefile b/cpp/Makefile index 7cc66b2..4a3677e 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -1,11 +1,11 @@ -CXXFLAGS = -I.. -I. -Wall -g -#CXXFLAGS = -I.. -I. -Wall -g -O4 -LDFLAGS = -L. +#CXXFLAGS = -I.. -I. -Wall -g +CXXFLAGS = -I.. -I. -Wall -g -O4 +LDFLAGS = -L. $(CXXFLAGS) NEED_PREPROCESS = zone.hpp -all: test +all: test bench %.hpp: %.hpp.erb erb $< > $@ @@ -13,10 +13,13 @@ all: test test: $(NEED_PREPROCESS) unpack.o unpack_inline.o object.o zone.o test.o object.hpp unpack.hpp pack.hpp $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o object.o test.o -o $@ +bench: $(NEED_PREPROCESS) unpack.o unpack_inline.o object.o zone.o bench.o object.hpp unpack.hpp pack.hpp + $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o object.o bench.o -o $@ + .PHONY: clean clean: $(RM) unpack.o unpack_inline.o object.o zone.o - $(RM) test.o - $(RM) test + $(RM) test.o test + $(RM) bench.o bench $(RM) $(NEED_PREPROCESS) diff --git a/cpp/bench.cpp b/cpp/bench.cpp new file mode 100644 index 0000000..1a9f6f9 --- /dev/null +++ b/cpp/bench.cpp @@ -0,0 +1,181 @@ +#include +#include +#include +#include +#include +#include +#include + +static const unsigned int TASK_INT_NUM = 1<<24; +static const unsigned int TASK_STR_LEN = 1<<15; +//static const unsigned int TASK_INT_NUM = 1<<23; +//static const unsigned int TASK_STR_LEN = 1<<14; +static const void* TASK_STR_PTR; + + +class simple_timer { +public: + void reset() { gettimeofday(&m_timeval, NULL); } + void show_stat(size_t bufsz) + { + struct timeval endtime; + gettimeofday(&endtime, NULL); + double sec = (endtime.tv_sec - m_timeval.tv_sec) + + (double)(endtime.tv_usec - m_timeval.tv_usec) / 1000 / 1000; + std::cout << sec << " sec" << std::endl; + std::cout << (double(bufsz)/1024/1024) << " MB" << std::endl; + std::cout << (bufsz/sec/1024/1024*8) << " Mbps" << std::endl; + } +private: + timeval m_timeval; +}; + + +class simple_buffer { +public: + static const size_t DEFAULT_INITIAL_SIZE = 32*1024;//512*1024*1024*2; + + simple_buffer(size_t initial_size = DEFAULT_INITIAL_SIZE) : + m_storage((char*)malloc(initial_size)), + m_allocated(initial_size), + m_used(0) + { + if(!m_storage) { throw std::bad_alloc(); } + } + + ~simple_buffer() + { + free(m_storage); + } + +public: + inline simple_buffer& append(const char* buf, size_t len) + { + if(m_allocated - m_used < len) { + expand_buffer(len); + } + memcpy(m_storage + m_used, buf, len); + m_used += len; + return *this; + } + + void clear() + { + m_used = 0; + } + +private: + void expand_buffer(size_t req) + { + size_t nsize = m_allocated * 2; + size_t at_least = m_used + req; + while(nsize < at_least) { nsize *= 2; } + char* tmp = (char*)realloc(m_storage, nsize); + if(!tmp) { throw std::bad_alloc(); } + m_storage = tmp; + m_allocated = nsize; + } + +public: + size_t size() const { return m_used; } + const char* data() const { return m_storage; } + +private: + char* m_storage; + size_t m_allocated; + size_t m_used; +}; + + +void bench_msgpack_int() +{ + simple_buffer buf; + simple_timer timer; + + std::cout << "----" << std::endl; + std::cout << "pack integer" << std::endl; + + timer.reset(); + { + msgpack::packer pk(buf); + pk.pack_array(TASK_INT_NUM); + for(unsigned int i=0; i < TASK_INT_NUM; ++i) { + pk.pack_unsigned_int(i); + } + } + timer.show_stat(buf.size()); + + std::cout << "----" << std::endl; + std::cout << "unpack integer" << std::endl; + + msgpack::zone z; + msgpack::object obj; + + timer.reset(); + { + obj = msgpack::unpack(buf.data(), buf.size(), z); + } + timer.show_stat(buf.size()); + + std::cout << "----" << std::endl; + std::cout << "dynamic pack integer" << std::endl; + + buf.clear(); + + timer.reset(); + msgpack::pack(buf, obj); + timer.show_stat(buf.size()); +} + +void bench_msgpack_str() +{ + simple_buffer buf; + simple_timer timer; + + std::cout << "----" << std::endl; + std::cout << "pack string" << std::endl; + + timer.reset(); + { + msgpack::packer pk(buf); + pk.pack_array(TASK_STR_LEN); + for(unsigned int i=0; i < TASK_STR_LEN; ++i) { + pk.pack_raw(TASK_STR_PTR, i); + } + } + timer.show_stat(buf.size()); + + std::cout << "----" << std::endl; + std::cout << "unpack string" << std::endl; + + msgpack::zone z; + msgpack::object obj; + + timer.reset(); + { + obj = msgpack::unpack(buf.data(), buf.size(), z); + } + timer.show_stat(buf.size()); + + std::cout << "----" << std::endl; + std::cout << "dynamic pack string" << std::endl; + + buf.clear(); + + timer.reset(); + msgpack::pack(buf, obj); + timer.show_stat(buf.size()); +} + +int main(void) +{ + void* str = malloc(TASK_STR_LEN); + memset(str, 'a', TASK_STR_LEN); + TASK_STR_PTR = str; + + bench_msgpack_int(); + bench_msgpack_str(); + + return 0; +} + diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index d484593..807ded9 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -85,8 +85,8 @@ void unpacker::expand_buffer(size_t len) void* tmp = malloc(next_size); if(!tmp) { throw std::bad_alloc(); } memcpy(tmp, m_buffer, m_used); - free(m_buffer); + m_buffer = tmp; m_free = next_size - m_used; From 97bc0441b1204b3eb38f9cfc03f90e5ee675ab9a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0011/1172] lang/c/msgpack: C++ binding: enlarged chunk size of zone git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@58 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/unpack.cpp | 4 ++-- cpp/unpack.hpp | 3 +++ cpp/zone.cpp | 25 ++++++++++++++++--------- cpp/zone.hpp.erb | 11 +++++------ 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 807ded9..6a6a4e8 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -78,7 +78,7 @@ void unpacker::expand_buffer(size_t len) if(m_off == 0) { size_t next_size; if(m_free != 0) { next_size = m_free * 2; } - else { next_size = MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE; } + else { next_size = UNPACKER_INITIAL_BUFFER_SIZE; } while(next_size < len + m_used) { next_size *= 2; } // FIXME realloc? @@ -91,7 +91,7 @@ void unpacker::expand_buffer(size_t len) m_free = next_size - m_used; } else { - size_t next_size = MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE; + size_t next_size = UNPACKER_INITIAL_BUFFER_SIZE; while(next_size < len + m_used - m_off) { next_size *= 2; } void* tmp = malloc(next_size); diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index ae536c9..9bda544 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -12,6 +12,9 @@ namespace msgpack { +static const size_t UNPACKER_INITIAL_BUFFER_SIZE = MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE; + + struct unpack_error : public std::runtime_error { unpack_error(const std::string& msg) : std::runtime_error(msg) { } diff --git a/cpp/zone.cpp b/cpp/zone.cpp index de2de22..ea4c7e4 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -5,10 +5,17 @@ namespace msgpack { void* zone::alloc() { - if(m_used >= m_pool.size()*MSGPACK_ZONE_CHUNK_SIZE) { - m_pool.push_back(new chunk_t()); + if(m_pool.size()*ZONE_CHUNK_SIZE <= m_used) { + cell_t* chunk = (cell_t*)malloc(sizeof(cell_t)*ZONE_CHUNK_SIZE); + if(!chunk) { throw std::bad_alloc(); } + try { + m_pool.push_back(chunk); + } catch (...) { + free(chunk); + throw; + } } - void* data = m_pool[m_used/MSGPACK_ZONE_CHUNK_SIZE]->cells[m_used%MSGPACK_ZONE_CHUNK_SIZE].data; + void* data = m_pool[m_used/ZONE_CHUNK_SIZE][m_used%ZONE_CHUNK_SIZE].data; ++m_used; return data; } @@ -16,21 +23,21 @@ void* zone::alloc() void zone::clear() { if(!m_pool.empty()) { - for(size_t b=0; b < m_used/MSGPACK_ZONE_CHUNK_SIZE; ++b) { - cell_t* c(m_pool[b]->cells); - for(size_t e=0; e < MSGPACK_ZONE_CHUNK_SIZE; ++e) { + for(size_t b=0; b < m_used/ZONE_CHUNK_SIZE; ++b) { + cell_t* c(m_pool[b]); + for(size_t e=0; e < ZONE_CHUNK_SIZE; ++e) { reinterpret_cast(c[e].data)->~object_class(); } } - cell_t* c(m_pool.back()->cells); - for(size_t e=0; e < m_used%MSGPACK_ZONE_CHUNK_SIZE; ++e) { + cell_t* c(m_pool.back()); + for(size_t e=0; e < m_used%ZONE_CHUNK_SIZE; ++e) { reinterpret_cast(c[e].data)->~object_class(); } for(pool_t::iterator it(m_pool.begin()), it_end(m_pool.end()); it != it_end; ++it) { - delete *it; + free(*it); } m_pool.clear(); } diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 40ce694..5c0e0d8 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -8,12 +8,15 @@ #include #ifndef MSGPACK_ZONE_CHUNK_SIZE -#define MSGPACK_ZONE_CHUNK_SIZE 64 +#define MSGPACK_ZONE_CHUNK_SIZE 8*1024 #endif namespace msgpack { +static const size_t ZONE_CHUNK_SIZE = MSGPACK_ZONE_CHUNK_SIZE; + + class zone { public: zone() : m_used(0) { } @@ -110,11 +113,7 @@ private: char data[MAX_OBJECT_SIZE]; }; - struct chunk_t { - cell_t cells[MSGPACK_ZONE_CHUNK_SIZE]; - }; - - typedef std::vector pool_t; + typedef std::vector pool_t; pool_t m_pool; From b5f13e7d26e88f4c12675a14b62dda3cd3bc9598 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0012/1172] lang/c/msgpack: autotoolized git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@59 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- AUTHORS | 0 ChangeLog | 0 Makefile.am | 7 +++ NEWS | 0 bootstrap | 119 +++++++++++++++++++++++++++++++++++++++++++++++ c/AUTHORS | 0 c/ChangeLog | 0 c/Makefile | 17 ------- c/Makefile.am | 15 ++++++ c/NEWS | 0 c/README | 0 c/bootstrap | 3 ++ c/configure.in | 11 +++++ configure.in | 10 ++++ cpp/AUTHORS | 0 cpp/ChangeLog | 0 cpp/Makefile | 25 ---------- cpp/Makefile.am | 20 ++++++++ cpp/NEWS | 0 cpp/README | 0 cpp/bootstrap | 3 ++ cpp/configure.in | 14 ++++++ ruby/bench.rb | 22 ++++++--- 23 files changed, 218 insertions(+), 48 deletions(-) create mode 100644 AUTHORS create mode 100644 ChangeLog create mode 100644 Makefile.am create mode 100644 NEWS create mode 100755 bootstrap create mode 100644 c/AUTHORS create mode 100644 c/ChangeLog delete mode 100644 c/Makefile create mode 100644 c/Makefile.am create mode 100644 c/NEWS create mode 100644 c/README create mode 100755 c/bootstrap create mode 100644 c/configure.in create mode 100644 configure.in create mode 100644 cpp/AUTHORS create mode 100644 cpp/ChangeLog delete mode 100644 cpp/Makefile create mode 100644 cpp/Makefile.am create mode 100644 cpp/NEWS create mode 100644 cpp/README create mode 100755 cpp/bootstrap create mode 100644 cpp/configure.in diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..e69de29 diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..e69de29 diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..9d1282d --- /dev/null +++ b/Makefile.am @@ -0,0 +1,7 @@ +SUBDIRS = c cpp + +nobase_include_HEADERS = \ + msgpack/pack/inline_impl.h \ + msgpack/unpack/inline_context.h \ + msgpack/unpack/inline_impl.h + diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..e69de29 diff --git a/bootstrap b/bootstrap new file mode 100755 index 0000000..954fa89 --- /dev/null +++ b/bootstrap @@ -0,0 +1,119 @@ +#!/bin/sh +# vim:ts=4:sw=4 +# Calls autotools to build configure script and Makefile.in. +# Generated automatically using bootstrapper 0.2.1 +# http://bootstrapper.sourceforge.net/ +# +# Copyright (C) 2002 Anthony Ventimiglia +# +# This bootstrap script is free software; you can redistribute +# it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# +# Calls proper programs to create configure script and Makefile.in files. +# if run with the --clean option, bootstrap removes files it generates. To +# clean all autogenerated files (eg: for cvs imports) first run +# make distclean, then bootstrap --clean +# see bootstrapper(1) for more infor + + +if test x"$1" = x"--help"; then + echo "$0: automatic bootstrapping utility for GNU Autotools" + echo " cleans up old autogenerated files and runs autoconf," + echo " automake and aclocal on local directory" + echo + echo " --clean clean up auto-generated files without" + echo " creating new scripts" + echo + exit 0 +fi + + +if [ -z "$NO_NEST" ];then + cd c && ./bootstrap $@; cd .. + cd cpp && ./bootstrap $@; cd .. +fi + + +ACLOCAL="aclocal" +ACLOCAL_FILES="aclocal.m4" +ALWAYS_CLEAN="config.status config.log config.cache libtool" +AUTOCONF="autoconf" +AUTOCONF_FILES="configure" +AUTOHEADER="autoheader" +AUTOHEADER_FILES="" +AUTOMAKE="automake --add-missing --copy" +AUTOMAKE_FILES="config.sub stamp-h.in ltmain.sh missing mkinstalldirs install-sh config.guess" +CONFIG_AUX_DIR="." +CONFIG_FILES="stamp-h ltconfig" +CONFIG_HEADER="" +if [ x`uname` = x"Darwin" ]; then + LIBTOOLIZE="glibtoolize --force --copy" +else + LIBTOOLIZE="libtoolize --force --copy" +fi +LIBTOOLIZE_FILES="config.sub ltmain.sh config.guess" +RM="rm" +SUBDIRS="[]" + + +# These are files created by configure, so we'll always clean them +for i in $ALWAYS_CLEAN; do + test -f $i && \ + $RM $i +done + +if test x"$1" = x"--clean"; then + # + #Clean Files left by previous bootstrap run + # + if test -n "$CONFIG_AUX_DIR"; + then CONFIG_AUX_DIR="$CONFIG_AUX_DIR/" + fi + # Clean Libtoolize generated files + for cf in $LIBTOOLIZE_FILES; do + cf="$CONFIG_AUX_DIR$cf" + test -f $cf && \ + $RM $cf + done + #aclocal.m4 created by aclocal + test -f $ACLOCAL_FILES && $RM $ACLOCAL_FILES + #Clean Autoheader Generated files + for cf in $AUTOHEADER_FILES; do + cf=$CONFIG_AUX_DIR$cf + test -f $cf && \ + $RM $cf + done + # remove config header (Usaually config.h) + test -n "$CONFIG_HEADER" && test -f $CONFIG_HEADER && $RM $CONFIG_HEADER + #Clean Automake generated files + for cf in $AUTOMAKE_FILES; do + cf=$CONFIG_AUX_DIR$cf + test -f $cf && \ + $RM $cf + done + for i in $SUBDIRS; do + test -f $i/Makefile.in && \ + $RM $i/Makefile.in + done + #Autoconf generated files + for cf in $AUTOCONF_FILES; do + test -f $cf && \ + $RM $cf + done + for cf in $CONFIG_FILES; do + cf="$CONFIG_AUX_DIR$cf" + test -f $cf && \ + $RM $cf + done +else + $LIBTOOLIZE + $ACLOCAL + $AUTOHEADER + $AUTOMAKE + $AUTOCONF +fi + + diff --git a/c/AUTHORS b/c/AUTHORS new file mode 100644 index 0000000..e69de29 diff --git a/c/ChangeLog b/c/ChangeLog new file mode 100644 index 0000000..e69de29 diff --git a/c/Makefile b/c/Makefile deleted file mode 100644 index d4b2226..0000000 --- a/c/Makefile +++ /dev/null @@ -1,17 +0,0 @@ - -CFLAGS = -I.. -Wall -g -O4 -LDFLAGS = -L. - -all: bench - -bench: pack.o unpack.o unpack_inline.o bench.o pack.h pack_inline.h unpack.h unpack_context.h - $(CC) $(LDFLAGS) unpack.o unpack_inline.o pack.o bench.o -lyajl_s -o $@ - -bench_inline: pack.o bench_inline.o pack.h pack_inline.h - $(CC) $(LDFLAGS) pack.o bench_inline.o -lyajl_s -o $@ - -.PHONY: clean -clean: - $(RM) unpack.o unpack_inline.o pack.o test.o bench.o bench_inline.o - $(RM) bench bench_inline - diff --git a/c/Makefile.am b/c/Makefile.am new file mode 100644 index 0000000..15cf753 --- /dev/null +++ b/c/Makefile.am @@ -0,0 +1,15 @@ +lib_LTLIBRARIES = libmsgpackc.la + +libmsgpackc_la_SOURCES = \ + pack.c \ + unpack.c \ + unpack_inline.c + +nobase_include_HEADERS = \ + msgpack/pack.h \ + msgpack/unpack.h + +noinst_HEADERS = \ + pack_inline.h \ + unpack_context.h + diff --git a/c/NEWS b/c/NEWS new file mode 100644 index 0000000..e69de29 diff --git a/c/README b/c/README new file mode 100644 index 0000000..e69de29 diff --git a/c/bootstrap b/c/bootstrap new file mode 100755 index 0000000..6a1e814 --- /dev/null +++ b/c/bootstrap @@ -0,0 +1,3 @@ +#!/bin/sh +NO_NEST=1 +source ../bootstrap diff --git a/c/configure.in b/c/configure.in new file mode 100644 index 0000000..0f37460 --- /dev/null +++ b/c/configure.in @@ -0,0 +1,11 @@ +AC_INIT(pack.h) +AM_INIT_AUTOMAKE(msgpackc, 0.1) +AC_CONFIG_HEADER(config.h) + +AC_PROG_CC +AC_PROG_LIBTOOL + +CFLAGS="-O4 $CFLAGS -Wall -I.." + +AC_OUTPUT([Makefile]) + diff --git a/configure.in b/configure.in new file mode 100644 index 0000000..941168c --- /dev/null +++ b/configure.in @@ -0,0 +1,10 @@ +AC_INIT(msgpack/unpack/inline_impl.h) +AM_INIT_AUTOMAKE(msgpack, 0.1) +AC_CONFIG_HEADER(config.h) + +AC_PROG_LIBTOOL +#AC_CHECK_PROG(RUBY, ruby, ruby, [$PATH]) + +AC_CONFIG_SUBDIRS([c cpp]) +AC_OUTPUT([Makefile]) + diff --git a/cpp/AUTHORS b/cpp/AUTHORS new file mode 100644 index 0000000..e69de29 diff --git a/cpp/ChangeLog b/cpp/ChangeLog new file mode 100644 index 0000000..e69de29 diff --git a/cpp/Makefile b/cpp/Makefile deleted file mode 100644 index 4a3677e..0000000 --- a/cpp/Makefile +++ /dev/null @@ -1,25 +0,0 @@ - -#CXXFLAGS = -I.. -I. -Wall -g -CXXFLAGS = -I.. -I. -Wall -g -O4 -LDFLAGS = -L. $(CXXFLAGS) - -NEED_PREPROCESS = zone.hpp - -all: test bench - -%.hpp: %.hpp.erb - erb $< > $@ - -test: $(NEED_PREPROCESS) unpack.o unpack_inline.o object.o zone.o test.o object.hpp unpack.hpp pack.hpp - $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o object.o test.o -o $@ - -bench: $(NEED_PREPROCESS) unpack.o unpack_inline.o object.o zone.o bench.o object.hpp unpack.hpp pack.hpp - $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o object.o bench.o -o $@ - -.PHONY: clean -clean: - $(RM) unpack.o unpack_inline.o object.o zone.o - $(RM) test.o test - $(RM) bench.o bench - $(RM) $(NEED_PREPROCESS) - diff --git a/cpp/Makefile.am b/cpp/Makefile.am new file mode 100644 index 0000000..664f1ed --- /dev/null +++ b/cpp/Makefile.am @@ -0,0 +1,20 @@ +lib_LTLIBRARIES = libmsgpack.la + +libmsgpack_la_SOURCES = \ + object.cpp \ + unpack.cpp \ + unpack_inline.cpp \ + zone.cpp + +nobase_include_HEADERS = \ + msgpack/pack.hpp \ + msgpack/unpack.hpp \ + msgpack/object.hpp \ + msgpack/zone.hpp + +noinst_HEADERS = \ + unpack_context.hpp + +zone.hpp: zone.hpp.erb + erb $< > $@ + diff --git a/cpp/NEWS b/cpp/NEWS new file mode 100644 index 0000000..e69de29 diff --git a/cpp/README b/cpp/README new file mode 100644 index 0000000..e69de29 diff --git a/cpp/bootstrap b/cpp/bootstrap new file mode 100755 index 0000000..6a1e814 --- /dev/null +++ b/cpp/bootstrap @@ -0,0 +1,3 @@ +#!/bin/sh +NO_NEST=1 +source ../bootstrap diff --git a/cpp/configure.in b/cpp/configure.in new file mode 100644 index 0000000..a94738a --- /dev/null +++ b/cpp/configure.in @@ -0,0 +1,14 @@ +AC_INIT(object.hpp) +AM_INIT_AUTOMAKE(msgpack, 0.1) +AC_CONFIG_HEADER(config.h) + +AC_PROG_CXX +AC_PROG_LIBTOOL +AC_CHECK_PROG(ERB, erb, erb, [$PATH]) + +AC_CHECK_LIB(stdc++, main) + +CXXFLAGS="-O4 $CXXFLAGS -Wall -I.." + +AC_OUTPUT([Makefile]) + diff --git a/ruby/bench.rb b/ruby/bench.rb index 1e6e27b..3b0b2ae 100644 --- a/ruby/bench.rb +++ b/ruby/bench.rb @@ -9,9 +9,9 @@ end ary = [] i = 0 -while i < (1<<23) - ary << (1<<23) - #ary << i +while i < (1<<24) + #ary << (1<<24) + ary << i i += 1 end @@ -23,10 +23,13 @@ a = Time.now packed = MessagePack::pack(ary) b = Time.now show10(packed) -puts "#{b-a} sec." +sec = b - a +puts "#{sec} sec." +puts "#{packed.length.to_f / sec / 1024 / 1024 * 8} Mbps" GC.start +=begin puts "----" puts "JSON" a = Time.now @@ -37,6 +40,7 @@ puts "#{b-a} sec." ary = nil GC.start +=end puts "----" @@ -44,17 +48,23 @@ puts "MessagePack" a = Time.now ary = MessagePack::unpack(packed) b = Time.now -puts "#{b-a} sec." +sec = b - a +puts "#{sec} sec." +puts "#{packed.length.to_f / sec / 1024 / 1024 * 8} Mbps" + +p ary.size +p (1<<24) ary = nil GC.start +=begin puts "----" puts "JSON" a = Time.now ary = JSON::load(json) b = Time.now puts "#{b-a} sec." - +=end From a721837ed0c1b9783deb72a09d51dfad79411262 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:57 +0000 Subject: [PATCH 0013/1172] lang/c/msgpack: added license notifications git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@60 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- AUTHORS | 1 + COPYING | 14 ++++++++++++++ c/AUTHORS | 1 + c/COPYING | 14 ++++++++++++++ c/Makefile.am | 3 +++ c/README | 21 +++++++++++++++++++++ c/configure.in | 2 +- c/msgpack.h | 19 +++++++++++++++++++ c/pack.h | 17 +++++++++++++++++ c/pack_inline.h | 17 +++++++++++++++++ c/unpack.c | 2 +- configure.in | 2 +- cpp/AUTHORS | 1 + cpp/COPYING | 14 ++++++++++++++ cpp/Makefile.am | 7 ++++++- cpp/README | 21 +++++++++++++++++++++ cpp/configure.in | 2 +- cpp/msgpack.hpp | 21 +++++++++++++++++++++ cpp/object.cpp | 17 +++++++++++++++++ cpp/object.hpp | 17 +++++++++++++++++ cpp/pack.hpp | 17 +++++++++++++++++ cpp/test.cpp | 5 +++-- cpp/unpack.cpp | 17 +++++++++++++++++ cpp/unpack.hpp | 17 +++++++++++++++++ cpp/unpack_context.hpp | 17 +++++++++++++++++ cpp/unpack_inline.cpp | 17 +++++++++++++++++ cpp/zone.cpp | 17 +++++++++++++++++ cpp/zone.hpp.erb | 17 +++++++++++++++++ ruby/gem/Manifest.txt | 1 - ruby/gem/lib/msgpack/version.rb | 4 ++-- ruby/{gem.sh => gengem.sh} | 1 - 31 files changed, 332 insertions(+), 11 deletions(-) create mode 100644 COPYING create mode 100644 c/COPYING create mode 100644 c/msgpack.h create mode 100644 cpp/COPYING create mode 100644 cpp/msgpack.hpp rename ruby/{gem.sh => gengem.sh} (90%) diff --git a/AUTHORS b/AUTHORS index e69de29..ababacb 100644 --- a/AUTHORS +++ b/AUTHORS @@ -0,0 +1 @@ +FURUHASHI Sadayuki diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..5f100cd --- /dev/null +++ b/COPYING @@ -0,0 +1,14 @@ +Copyright (C) 2008 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/c/AUTHORS b/c/AUTHORS index e69de29..ababacb 100644 --- a/c/AUTHORS +++ b/c/AUTHORS @@ -0,0 +1 @@ +FURUHASHI Sadayuki diff --git a/c/COPYING b/c/COPYING new file mode 100644 index 0000000..5f100cd --- /dev/null +++ b/c/COPYING @@ -0,0 +1,14 @@ +Copyright (C) 2008 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/c/Makefile.am b/c/Makefile.am index 15cf753..af8f3ad 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -6,6 +6,7 @@ libmsgpackc_la_SOURCES = \ unpack_inline.c nobase_include_HEADERS = \ + msgpack.h \ msgpack/pack.h \ msgpack/unpack.h @@ -13,3 +14,5 @@ noinst_HEADERS = \ pack_inline.h \ unpack_context.h +libmsgpackc_la_LDFLAGS = -version-info 0:0:0 + diff --git a/c/README b/c/README index e69de29..609b67d 100644 --- a/c/README +++ b/c/README @@ -0,0 +1,21 @@ +MessagePack for C +----------------- +MessagePack is a binary-based efficient data interchange format. + + + +Copyright (C) 2008 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + diff --git a/c/configure.in b/c/configure.in index 0f37460..4bc7e3e 100644 --- a/c/configure.in +++ b/c/configure.in @@ -1,5 +1,5 @@ AC_INIT(pack.h) -AM_INIT_AUTOMAKE(msgpackc, 0.1) +AM_INIT_AUTOMAKE(msgpackc, 0.1.0) AC_CONFIG_HEADER(config.h) AC_PROG_CC diff --git a/c/msgpack.h b/c/msgpack.h new file mode 100644 index 0000000..7a15f71 --- /dev/null +++ b/c/msgpack.h @@ -0,0 +1,19 @@ +/* + * MessagePack for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/pack.h" +#include "msgpack/unpack.h" diff --git a/c/pack.h b/c/pack.h index 8107d9f..3144f37 100644 --- a/c/pack.h +++ b/c/pack.h @@ -1,3 +1,20 @@ +/* + * MessagePack packing routine for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef MSGPACK_PACK_H__ #define MSGPACK_PACK_H__ diff --git a/c/pack_inline.h b/c/pack_inline.h index b1f9e63..acd96fa 100644 --- a/c/pack_inline.h +++ b/c/pack_inline.h @@ -1,3 +1,20 @@ +/* + * MessagePack packing routine for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef PACK_INLINE_H__ #define PACK_INLINE_H__ diff --git a/c/unpack.c b/c/unpack.c index 9fe3695..012888e 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -1,5 +1,5 @@ /* - * MessagePack packing routine for C + * MessagePack unpacking routine for C * * Copyright (C) 2008 FURUHASHI Sadayuki * diff --git a/configure.in b/configure.in index 941168c..9b6f6d4 100644 --- a/configure.in +++ b/configure.in @@ -1,5 +1,5 @@ AC_INIT(msgpack/unpack/inline_impl.h) -AM_INIT_AUTOMAKE(msgpack, 0.1) +AM_INIT_AUTOMAKE(msgpack, 0.1.0) AC_CONFIG_HEADER(config.h) AC_PROG_LIBTOOL diff --git a/cpp/AUTHORS b/cpp/AUTHORS index e69de29..ababacb 100644 --- a/cpp/AUTHORS +++ b/cpp/AUTHORS @@ -0,0 +1 @@ +FURUHASHI Sadayuki diff --git a/cpp/COPYING b/cpp/COPYING new file mode 100644 index 0000000..5f100cd --- /dev/null +++ b/cpp/COPYING @@ -0,0 +1,14 @@ +Copyright (C) 2008 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 664f1ed..b9ecb1d 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -7,6 +7,7 @@ libmsgpack_la_SOURCES = \ zone.cpp nobase_include_HEADERS = \ + msgpack.hpp \ msgpack/pack.hpp \ msgpack/unpack.hpp \ msgpack/object.hpp \ @@ -15,6 +16,10 @@ nobase_include_HEADERS = \ noinst_HEADERS = \ unpack_context.hpp -zone.hpp: zone.hpp.erb +msgpack/zone.hpp: msgpack/zone.hpp.erb erb $< > $@ +MOSTLYCLEANFILES = zone.hpp + +libmsgpack_la_LDFLAGS = -version-info 0:0:0 + diff --git a/cpp/README b/cpp/README index e69de29..eceff1b 100644 --- a/cpp/README +++ b/cpp/README @@ -0,0 +1,21 @@ +MessagePack for C++ +------------------- +MessagePack is a binary-based efficient data interchange format. + + + +Copyright (C) 2008 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + diff --git a/cpp/configure.in b/cpp/configure.in index a94738a..e9e21ea 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -1,5 +1,5 @@ AC_INIT(object.hpp) -AM_INIT_AUTOMAKE(msgpack, 0.1) +AM_INIT_AUTOMAKE(msgpack, 0.1.0) AC_CONFIG_HEADER(config.h) AC_PROG_CXX diff --git a/cpp/msgpack.hpp b/cpp/msgpack.hpp new file mode 100644 index 0000000..9f635d0 --- /dev/null +++ b/cpp/msgpack.hpp @@ -0,0 +1,21 @@ +// +// MessagePack for C++ +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#include "msgpack/object.hpp" +#include "msgpack/zone.hpp" +#include "msgpack/pack.hpp" +#include "msgpack/unpack.hpp" diff --git a/cpp/object.cpp b/cpp/object.cpp index 947e8dd..d6815ac 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ dynamic typed objects +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #include "msgpack/object.hpp" #include "msgpack/pack.hpp" diff --git a/cpp/object.hpp b/cpp/object.hpp index f029e32..b1f5369 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ dynamic typed objects +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #ifndef MSGPACK_OBJECT_HPP__ #define MSGPACK_OBJECT_HPP__ diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 89f8cee..ac7bb83 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ serializing routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #ifndef MSGPACK_PACK_HPP__ #define MSGPACK_PACK_HPP__ diff --git a/cpp/test.cpp b/cpp/test.cpp index e236d0f..7c91fbd 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -1,7 +1,8 @@ #include #include -#include -#include +//#include +//#include +#include #include #include diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 6a6a4e8..a0128bb 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ deserializing routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #include "msgpack/unpack.hpp" #include "unpack_context.hpp" #include diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 9bda544..452a231 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ deserializing routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #ifndef MSGPACK_UNPACK_HPP__ #define MSGPACK_UNPACK_HPP__ diff --git a/cpp/unpack_context.hpp b/cpp/unpack_context.hpp index 49a93e4..59bd872 100644 --- a/cpp/unpack_context.hpp +++ b/cpp/unpack_context.hpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ deserializing routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #ifndef UNPACK_CONTEXT_HPP__ #define UNPACK_CONTEXT_HPP__ diff --git a/cpp/unpack_inline.cpp b/cpp/unpack_inline.cpp index 9002327..37e0b66 100644 --- a/cpp/unpack_inline.cpp +++ b/cpp/unpack_inline.cpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ deserializing routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #include "unpack_context.hpp" diff --git a/cpp/zone.cpp b/cpp/zone.cpp index ea4c7e4..22be759 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ memory pool +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #include "zone.hpp" namespace msgpack { diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 5c0e0d8..ac08595 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -1,3 +1,20 @@ +// +// MessagePack for C++ memory pool +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #ifndef MSGPACK_ZONE_HPP__ #define MSGPACK_ZONE_HPP__ #include diff --git a/ruby/gem/Manifest.txt b/ruby/gem/Manifest.txt index fac8555..299a4e0 100644 --- a/ruby/gem/Manifest.txt +++ b/ruby/gem/Manifest.txt @@ -13,7 +13,6 @@ ext/unpack.c ext/unpack.h ext/unpack_context.h ext/unpack_inline.c -msgpack/pack/inline_context.h msgpack/pack/inline_impl.h msgpack/unpack/inline_context.h msgpack/unpack/inline_impl.h diff --git a/ruby/gem/lib/msgpack/version.rb b/ruby/gem/lib/msgpack/version.rb index 44d1dc0..c65972f 100644 --- a/ruby/gem/lib/msgpack/version.rb +++ b/ruby/gem/lib/msgpack/version.rb @@ -1,8 +1,8 @@ module MessagePack module VERSION #:nodoc: MAJOR = 0 - MINOR = 0 - TINY = 1 + MINOR = 1 + TINY = 0 STRING = [MAJOR, MINOR, TINY].join('.') end diff --git a/ruby/gem.sh b/ruby/gengem.sh similarity index 90% rename from ruby/gem.sh rename to ruby/gengem.sh index 9d9f429..4f8623c 100755 --- a/ruby/gem.sh +++ b/ruby/gengem.sh @@ -10,7 +10,6 @@ cp unpack.h gem/ext/ cp unpack_context.h gem/ext/ cp unpack_inline.c gem/ext/ cp ../README gem/README.txt -cp ../msgpack/pack/inline_context.h gem/msgpack/pack/ cp ../msgpack/pack/inline_impl.h gem/msgpack/pack/ cp ../msgpack/unpack/inline_context.h gem/msgpack/unpack/ cp ../msgpack/unpack/inline_impl.h gem/msgpack/unpack/ From 4d13f614b6839ac4ca78f5526ab79e911f33ea65 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:57 +0000 Subject: [PATCH 0014/1172] lang/c/msgpack: optimize zone::alloc() git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@61 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.hpp | 76 ++++++++++++++++++++++++------------------------ cpp/unpack.cpp | 11 ++++--- cpp/zone.cpp | 32 ++++++++++---------- cpp/zone.hpp.erb | 40 +++++++++++++++---------- 4 files changed, 86 insertions(+), 73 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index b1f5369..959d84e 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -69,28 +69,28 @@ class dynamic_packer; struct object_class { virtual ~object_class() {} - virtual bool isnil() const { return false; } - virtual bool xbool() const { throw cast_error(); } - virtual uint8_t xu8() const { throw cast_error(); } - virtual uint16_t xu16() const { throw cast_error(); } - virtual uint32_t xu32() const { throw cast_error(); } - virtual uint64_t xu64() const { throw cast_error(); } - virtual int8_t xi8() const { throw cast_error(); } - virtual int16_t xi16() const { throw cast_error(); } - virtual int32_t xi32() const { throw cast_error(); } - virtual int64_t xi64() const { throw cast_error(); } - virtual float xfloat() const { throw cast_error(); } - virtual double xdouble() const { throw cast_error(); } - virtual raw xraw() { throw cast_error(); } - virtual array& xarray() { throw cast_error(); } - virtual map& xmap() { throw cast_error(); } - virtual const_raw xraw() const { throw cast_error(); } + virtual bool isnil () const { return false; } + virtual bool xbool () const { throw cast_error(); } + virtual uint8_t xu8 () const { throw cast_error(); } + virtual uint16_t xu16 () const { throw cast_error(); } + virtual uint32_t xu32 () const { throw cast_error(); } + virtual uint64_t xu64 () const { throw cast_error(); } + virtual int8_t xi8 () const { throw cast_error(); } + virtual int16_t xi16 () const { throw cast_error(); } + virtual int32_t xi32 () const { throw cast_error(); } + virtual int64_t xi64 () const { throw cast_error(); } + virtual float xfloat () const { throw cast_error(); } + virtual double xdouble() const { throw cast_error(); } + virtual raw xraw () { throw cast_error(); } + virtual array& xarray () { throw cast_error(); } + virtual map& xmap () { throw cast_error(); } + virtual const_raw xraw () const { throw cast_error(); } virtual const array& xarray() const { throw cast_error(); } - virtual const map& xmap() const { throw cast_error(); } + virtual const map& xmap () const { throw cast_error(); } virtual bool operator== (const object_class* x) const { return false; } - bool operator!= (const object_class* x) const { return !(this->operator==(x)); } virtual bool operator< (const object_class* x) const { throw cast_error(); } virtual bool operator> (const object_class* x) const { throw cast_error(); } + bool operator!= (const object_class* x) const { return !(this->operator==(x)); } virtual void pack(dynamic_packer& p) const = 0; operator bool() const { return xbool(); } // FIXME !isnil(); operator uint8_t() const { return xu8(); } @@ -126,24 +126,24 @@ struct object { object(object_class* v) : val(v) {} //object(object_class& v) : val(&v) {} ~object() {} - bool isnil() const { return val->isnil(); } - bool xbool() const { return val->xbool(); } - uint8_t xu8() const { return val->xu8(); } - uint16_t xu16() const { return val->xu16(); } - uint32_t xu32() const { return val->xu32(); } - uint64_t xu64() const { return val->xu64(); } - int8_t xi8() const { return val->xi8(); } - int16_t xi16() const { return val->xi16(); } - int32_t xi32() const { return val->xi32(); } - int64_t xi64() const { return val->xi64(); } - float xfloat() const { return val->xfloat(); } - double xdouble() const { return val->xdouble(); } - raw xraw() { return val->xraw(); } - array& xarray() { return val->xarray(); } - map& xmap() { return val->xmap(); } - const_raw xraw() const { return const_cast(val)->xraw(); } + bool isnil () const { return val->isnil(); } + bool xbool () const { return val->xbool(); } + uint8_t xu8 () const { return val->xu8(); } + uint16_t xu16 () const { return val->xu16(); } + uint32_t xu32 () const { return val->xu32(); } + uint64_t xu64 () const { return val->xu64(); } + int8_t xi8 () const { return val->xi8(); } + int16_t xi16 () const { return val->xi16(); } + int32_t xi32 () const { return val->xi32(); } + int64_t xi64 () const { return val->xi64(); } + float xfloat () const { return val->xfloat(); } + double xdouble() const { return val->xdouble(); } + raw xraw () { return val->xraw(); } + array& xarray () { return val->xarray(); } + map& xmap () { return val->xmap(); } + const_raw xraw () const { return const_cast(val)->xraw(); } const array& xarray() const { return const_cast(val)->xarray(); } - const map& xmap() const { return const_cast(val)->xmap(); } + const map& xmap () const { return const_cast(val)->xmap(); } bool operator== (object x) const { return val->operator== (x.val); } bool operator!= (object x) const { return val->operator!= (x.val); } bool operator< (object x) const { return val->operator< (x.val); } @@ -163,9 +163,9 @@ struct object { operator raw() { return val->operator raw(); } operator array&() { return val->operator array&(); } operator map&() { return val->operator map&(); } - operator raw() const { return val->operator raw(); } - operator array&() const { return val->operator array&(); } - operator map&() const { return val->operator map&(); } + operator const_raw() const { return val->operator const_raw(); } + operator const array&() const { return val->operator const array&(); } + operator const map&() const { return val->operator const map&(); } const object& inspect(std::ostream& s) const { val->inspect(s); return *this; } private: diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index a0128bb..7a0cd78 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -21,6 +21,7 @@ namespace msgpack { + struct unpacker::context { context(zone* z) { @@ -98,11 +99,13 @@ void unpacker::expand_buffer(size_t len) else { next_size = UNPACKER_INITIAL_BUFFER_SIZE; } while(next_size < len + m_used) { next_size *= 2; } - // FIXME realloc? - void* tmp = malloc(next_size); + void* tmp = realloc(m_buffer, next_size); if(!tmp) { throw std::bad_alloc(); } - memcpy(tmp, m_buffer, m_used); - free(m_buffer); + m_buffer = tmp; + //void* tmp = malloc(next_size); + //if(!tmp) { throw std::bad_alloc(); } + //memcpy(tmp, m_buffer, m_used); + //free(m_buffer); m_buffer = tmp; m_free = next_size - m_used; diff --git a/cpp/zone.cpp b/cpp/zone.cpp index 22be759..ff22fc3 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -20,34 +20,33 @@ namespace msgpack { -void* zone::alloc() +// FIXME custom allocator? + +void zone::expand_chunk() { - if(m_pool.size()*ZONE_CHUNK_SIZE <= m_used) { - cell_t* chunk = (cell_t*)malloc(sizeof(cell_t)*ZONE_CHUNK_SIZE); - if(!chunk) { throw std::bad_alloc(); } - try { - m_pool.push_back(chunk); - } catch (...) { - free(chunk); - throw; - } + cell_t* chunk = (cell_t*)malloc(sizeof(cell_t)*ZONE_CHUNK_SIZE); + if(!chunk) { throw std::bad_alloc(); } + try { + m_pool.push_back(chunk); + } catch (...) { + free(chunk); + throw; } - void* data = m_pool[m_used/ZONE_CHUNK_SIZE][m_used%ZONE_CHUNK_SIZE].data; - ++m_used; - return data; } void zone::clear() { if(!m_pool.empty()) { - for(size_t b=0; b < m_used/ZONE_CHUNK_SIZE; ++b) { + size_t base_size = m_used / ZONE_CHUNK_SIZE; + size_t extend_size = m_used % ZONE_CHUNK_SIZE; + for(size_t b=0; b < base_size; ++b) { cell_t* c(m_pool[b]); for(size_t e=0; e < ZONE_CHUNK_SIZE; ++e) { reinterpret_cast(c[e].data)->~object_class(); } } cell_t* c(m_pool.back()); - for(size_t e=0; e < m_used%ZONE_CHUNK_SIZE; ++e) { + for(size_t e=0; e < extend_size; ++e) { reinterpret_cast(c[e].data)->~object_class(); } @@ -60,7 +59,8 @@ void zone::clear() } m_used = 0; - for(user_finalizer_t::reverse_iterator it(m_user_finalizer.rbegin()), it_end(m_user_finalizer.rend()); + for(user_finalizer_t::reverse_iterator it(m_user_finalizer.rbegin()), + it_end(m_user_finalizer.rend()); it != it_end; ++it) { it->call(); diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index ac08595..d63fae8 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -25,7 +25,7 @@ #include #ifndef MSGPACK_ZONE_CHUNK_SIZE -#define MSGPACK_ZONE_CHUNK_SIZE 8*1024 +#define MSGPACK_ZONE_CHUNK_SIZE 1024 #endif namespace msgpack { @@ -44,19 +44,19 @@ public: void push_finalizer(void (*func)(void* obj, void* user), T* obj, void* user); public: - object_nil* nnil() { return new (alloc()) object_nil(); } - object_true* ntrue() { return new (alloc()) object_true(); } - object_false* nfalse() { return new (alloc()) object_false(); } - object_u8* nu8( uint8_t v) { return new (alloc()) object_u8(v); } - object_u16* nu16(uint16_t v) { return new (alloc()) object_u16(v); } - object_u32* nu32(uint32_t v) { return new (alloc()) object_u32(v); } - object_u64* nu64(uint64_t v) { return new (alloc()) object_u64(v); } - object_i8* ni8( int8_t v) { return new (alloc()) object_i8(v); } - object_i16* ni16( int16_t v) { return new (alloc()) object_i16(v); } - object_i32* ni32( int32_t v) { return new (alloc()) object_i32(v); } - object_i64* ni64( int64_t v) { return new (alloc()) object_i64(v); } - object_float* nfloat( float v) { return new (alloc()) object_float(v); } - object_double* ndouble( double v) { return new (alloc()) object_double(v); } + object_nil* nnil () { return new (alloc()) object_nil(); } + object_true* ntrue () { return new (alloc()) object_true(); } + object_false* nfalse () { return new (alloc()) object_false(); } + object_u8* nu8 (uint8_t v) { return new (alloc()) object_u8(v); } + object_u16* nu16 (uint16_t v) { return new (alloc()) object_u16(v); } + object_u32* nu32 (uint32_t v) { return new (alloc()) object_u32(v); } + object_u64* nu64 (uint64_t v) { return new (alloc()) object_u64(v); } + object_i8* ni8 (int8_t v) { return new (alloc()) object_i8(v); } + object_i16* ni16 (int16_t v) { return new (alloc()) object_i16(v); } + object_i32* ni32 (int32_t v) { return new (alloc()) object_i32(v); } + object_i64* ni64 (int64_t v) { return new (alloc()) object_i64(v); } + object_float* nfloat (float v) { return new (alloc()) object_float(v); } + object_double* ndouble(double v) { return new (alloc()) object_double(v); } object_raw_ref* nraw_ref(void* ptr, uint32_t len) { return new (alloc()) object_raw_ref(ptr, len); } @@ -149,7 +149,7 @@ private: user_finalizer_t m_user_finalizer; private: - void resize_pool(size_t n); + void expand_chunk(); public: static void finalize_free(void* obj, void* user) @@ -173,6 +173,16 @@ inline bool zone::empty() const return m_used == 0 && m_user_finalizer.empty(); } +inline void* zone::alloc() +{ + if(m_pool.size() <= m_used/ZONE_CHUNK_SIZE) { + expand_chunk(); + } + void* data = m_pool[m_used/ZONE_CHUNK_SIZE][m_used%ZONE_CHUNK_SIZE].data; + ++m_used; + return data; +} + } // namespace msgpack From a0a798d79e5c11bae1b0b6a94b25e0dee0c19b77 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:57 +0000 Subject: [PATCH 0015/1172] lang/c/msgpack: C++ binding: changed calback function of packer from Stream& append(const char*, size_t) to SomeType write(SomePointerType, SomeSizeType) git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@62 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/bench.cpp | 3 +-- cpp/pack.hpp | 47 ++++++++++++++++++++++++++++++++--------------- cpp/test.cpp | 5 +++-- cpp/unpack.cpp | 8 ++++---- cpp/unpack.hpp | 14 +++++++------- 5 files changed, 47 insertions(+), 30 deletions(-) diff --git a/cpp/bench.cpp b/cpp/bench.cpp index 1a9f6f9..649c0cc 100644 --- a/cpp/bench.cpp +++ b/cpp/bench.cpp @@ -49,14 +49,13 @@ public: } public: - inline simple_buffer& append(const char* buf, size_t len) + inline void write(const void* buf, size_t len) { if(m_allocated - m_used < len) { expand_buffer(len); } memcpy(m_storage + m_used, buf, len); m_used += len; - return *this; } void clear() diff --git a/cpp/pack.hpp b/cpp/pack.hpp index ac7bb83..94b5b9a 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -73,7 +73,7 @@ private: static void pack_string_impl(Stream& x, const char* b); static void pack_raw_impl(Stream& x, const void* b, size_t l); static void append_buffer(Stream& x, const unsigned char* buf, unsigned int len) - { x.append((const char*)buf, len); } + { x.write((const char*)buf, len); } private: Stream& m_stream; @@ -100,14 +100,13 @@ public: template dynamic_stream(Stream& s); public: - dynamic_stream& append(const char* buf, size_t len) - { (*m_function)(m_object, buf, len); return *this; } + void write(const char* buf, size_t len) + { (*m_function)(m_object, buf, len); } private: void* m_object; void (*m_function)(void* object, const char* buf, size_t len); private: - template - static void append_trampoline(void* object, const char* buf, size_t len); + struct write_trampoline; }; @@ -137,8 +136,6 @@ public: void pack_string(const char* b) { pack_string_impl(m_stream, b); } void pack_raw(const void* b, size_t l) { pack_raw_impl(m_stream, b, l); } -public: - private: static void pack_int_impl(dynamic_stream& x, int d); static void pack_unsigned_int_impl(dynamic_stream& x, unsigned int d); @@ -160,7 +157,7 @@ private: static void pack_string_impl(dynamic_stream& x, const char* b); static void pack_raw_impl(dynamic_stream& x, const void* b, size_t l); static void append_buffer(dynamic_stream& x, const unsigned char* buf, unsigned int len) - { x.append((const char*)buf, len); } + { x.write((const char*)buf, len); } private: dynamic_stream m_stream; @@ -181,17 +178,37 @@ private: template dynamic_packer::dynamic_packer(Stream& s) : m_stream(s) { } +struct dynamic_stream::write_trampoline { +private: + template + struct ret_type { + typedef R (*type)(void*, const char*, size_t); + }; + + template + static R trampoline(void* obj, const char* buf, size_t len) + { + return (reinterpret_cast(obj)->*MemFun)(buf, len); + } + +public: + template + static typename ret_type::type get(R (Stream::*func)(Ptr*, Sz)) + { + R (*f)(void*, const char*, size_t) = + &trampoline; + return f; + } +}; + template dynamic_stream::dynamic_stream(Stream& s) { m_object = reinterpret_cast(&s); - m_function = &dynamic_stream::append_trampoline; -} - -template -void dynamic_stream::append_trampoline(void* object, const char* buf, size_t len) -{ - (reinterpret_cast(object)->*MemFun)(buf, len); + m_function = reinterpret_cast( + write_trampoline::get(&Stream::write) + ); } diff --git a/cpp/test.cpp b/cpp/test.cpp index 7c91fbd..b63f9a1 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -28,9 +28,10 @@ public: } try { - std::string s; + std::stringstream s; msgpack::pack(s, o); - object ro = msgpack::unpack(s.data(), s.size(), m_zone); + std::string str(s.str()); + object ro = msgpack::unpack(str.data(), str.size(), m_zone); if(ro != o) { throw std::runtime_error("NOT MATCH"); } } catch (std::runtime_error& e) { std::cout << "** REUNPACK FAILED **" << std::endl; diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 7a0cd78..de33051 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -99,10 +99,10 @@ void unpacker::expand_buffer(size_t len) else { next_size = UNPACKER_INITIAL_BUFFER_SIZE; } while(next_size < len + m_used) { next_size *= 2; } - void* tmp = realloc(m_buffer, next_size); + char* tmp = (char*)realloc(m_buffer, next_size); if(!tmp) { throw std::bad_alloc(); } m_buffer = tmp; - //void* tmp = malloc(next_size); + //char* tmp = (char*)malloc(next_size); //if(!tmp) { throw std::bad_alloc(); } //memcpy(tmp, m_buffer, m_used); //free(m_buffer); @@ -114,9 +114,9 @@ void unpacker::expand_buffer(size_t len) size_t next_size = UNPACKER_INITIAL_BUFFER_SIZE; while(next_size < len + m_used - m_off) { next_size *= 2; } - void* tmp = malloc(next_size); + char* tmp = (char*)malloc(next_size); if(!tmp) { throw std::bad_alloc(); } - memcpy(tmp, ((char*)m_buffer)+m_off, m_used-m_off); + memcpy(tmp, m_buffer+m_off, m_used-m_off); try { m_zone->push_finalizer(&zone::finalize_free, NULL, m_buffer); diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 452a231..f5896f3 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -48,7 +48,7 @@ public: void reserve_buffer(size_t len); /*! 2. read data to the buffer() up to buffer_capacity() bytes */ - void* buffer(); + char* buffer(); size_t buffer_capacity() const; /*! 3. specify the number of bytes actually copied */ @@ -74,7 +74,7 @@ public: // Note that there are no parsed buffer when execute() returned true. /*! get address of buffer that is not parsed */ - void* nonparsed_buffer(); + char* nonparsed_buffer(); size_t nonparsed_size() const; /*! get the number of bytes that is already parsed */ @@ -90,7 +90,7 @@ private: struct context; context* m_ctx; - void* m_buffer; + char* m_buffer; size_t m_used; size_t m_free; size_t m_off; @@ -110,8 +110,8 @@ inline void unpacker::reserve_buffer(size_t len) expand_buffer(len); } -inline void* unpacker::buffer() - { return (void*)(((char*)m_buffer)+m_used); } +inline char* unpacker::buffer() + { return m_buffer + m_used; } inline size_t unpacker::buffer_capacity() const { return m_free; } @@ -123,8 +123,8 @@ inline void unpacker::buffer_consumed(size_t len) } -inline void* unpacker::nonparsed_buffer() - { return (void*)(((char*)m_buffer)+m_off); } +inline char* unpacker::nonparsed_buffer() + { return m_buffer + m_off; } inline size_t unpacker::nonparsed_size() const { return m_used - m_off; } From 1278eb3c63e2371283e07c0360b044728d72c704 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:57 +0000 Subject: [PATCH 0016/1172] lang/c/msgpack: fix types git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@63 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/bench.c | 2 +- c/bench_inline.c | 2 -- c/unpack.h | 2 +- c/unpack_inline.c | 2 +- cpp/bench.cpp | 6 ++-- cpp/object.cpp | 19 +++++----- cpp/object.hpp | 61 +++++++++++++++++---------------- cpp/pack.hpp | 4 +-- cpp/test.cpp | 6 ++-- cpp/unpack.cpp | 6 ++-- cpp/unpack.hpp | 4 +-- cpp/unpack_inline.cpp | 4 +-- cpp/zone.hpp.erb | 28 ++++++++++----- msgpack/unpack/callback.h | 2 +- msgpack/unpack/inline_context.h | 2 +- msgpack/unpack/inline_impl.h | 4 +-- ruby/unpack_inline.c | 2 +- 17 files changed, 85 insertions(+), 71 deletions(-) diff --git a/c/bench.c b/c/bench.c index 91f0c21..6b8466f 100644 --- a/c/bench.c +++ b/c/bench.c @@ -55,7 +55,7 @@ static void* unpack_array_start(void* data, unsigned int n) { return NULL; } static void unpack_array_item(void* data, void* c, void* o) { } static void* unpack_map_start(void* data, unsigned int n) { return NULL; } static void unpack_map_item(void* data, void* c, void* k, void* v) { } -static void* unpack_raw(void* data, const void* b, const void* p, size_t l) { /*printf("unpack raw %p %lu\n",p,l);*/ return NULL; } +static void* unpack_raw(void* data, const char* b, const char* p, unsigned int l) { /*printf("unpack raw %p %lu\n",p,l);*/ return NULL; } typedef struct { size_t allocated; diff --git a/c/bench_inline.c b/c/bench_inline.c index 5390319..bb3c431 100644 --- a/c/bench_inline.c +++ b/c/bench_inline.c @@ -56,8 +56,6 @@ static inline void* msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int3 static inline void* msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d) { return NULL; } static inline void* msgpack_unpack_float(msgpack_unpack_context* x, float d) { return NULL; } static inline void* msgpack_unpack_double(msgpack_unpack_context* x, double d) { return NULL; } -static inline void* msgpack_unpack_big_int(msgpack_unpack_context* x, const void* b, unsigned int l) { return NULL; } -static inline void* msgpack_unpack_big_float(msgpack_unpack_context* x, const void* b, unsigned int l) { return NULL; } static inline void* msgpack_unpack_nil(msgpack_unpack_context* x) { return NULL; } static inline void* msgpack_unpack_true(msgpack_unpack_context* x) { return NULL; } static inline void* msgpack_unpack_false(msgpack_unpack_context* x) { return NULL; } diff --git a/c/unpack.h b/c/unpack.h index 094328e..6367439 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -39,7 +39,7 @@ typedef struct { void (*unpack_array_item)(void* data, void* c, void* o); void* (*unpack_map_start)(void* data, unsigned int n); void (*unpack_map_item)(void* data, void* c, void* k, void* v); - void* (*unpack_raw)(void* data, const void* b, const void* p, size_t l); + void* (*unpack_raw)(void* data, const char* b, const char* p, unsigned int l); } msgpack_unpack_callback; typedef struct { diff --git a/c/unpack_inline.c b/c/unpack_inline.c index d758d3e..5282282 100644 --- a/c/unpack_inline.c +++ b/c/unpack_inline.c @@ -71,7 +71,7 @@ static inline void* msgpack_unpack_map_start(msgpack_unpack_t* x, unsigned int n static inline void msgpack_unpack_map_item(msgpack_unpack_t* x, void* c, void* k, void* v) { x->callback.unpack_map_item(x->data, c, k, v); } -static inline void* msgpack_unpack_raw(msgpack_unpack_t* x, const void* b, const void* p, size_t l) +static inline void* msgpack_unpack_raw(msgpack_unpack_t* x, const char* b, const char* p, unsigned int l) { return x->callback.unpack_raw(x->data, b, p, l); } diff --git a/cpp/bench.cpp b/cpp/bench.cpp index 649c0cc..4133a2e 100644 --- a/cpp/bench.cpp +++ b/cpp/bench.cpp @@ -10,7 +10,7 @@ static const unsigned int TASK_INT_NUM = 1<<24; static const unsigned int TASK_STR_LEN = 1<<15; //static const unsigned int TASK_INT_NUM = 1<<23; //static const unsigned int TASK_STR_LEN = 1<<14; -static const void* TASK_STR_PTR; +static const char* TASK_STR_PTR; class simple_timer { @@ -49,7 +49,7 @@ public: } public: - inline void write(const void* buf, size_t len) + inline void write(const char* buf, size_t len) { if(m_allocated - m_used < len) { expand_buffer(len); @@ -168,7 +168,7 @@ void bench_msgpack_str() int main(void) { - void* str = malloc(TASK_STR_LEN); + char* str = (char*)malloc(TASK_STR_LEN); memset(str, 'a', TASK_STR_LEN); TASK_STR_PTR = str; diff --git a/cpp/object.cpp b/cpp/object.cpp index d6815ac..4068007 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -320,29 +320,30 @@ FLOAT_OBJECT(double) EXTRA \ bool object_##NAME::operator== (const object_class* x) const \ try { \ - const_raw xr(x->xraw()); \ + raw xr(x->xraw()); \ return len == xr.len && (ptr == xr.ptr || memcmp(ptr, xr.ptr, len) == 0); \ } catch (type_error&) { return false; } \ bool object_##NAME::operator< (const object_class* x) const { \ - const_raw xr(x->xraw()); \ + raw xr(x->xraw()); \ if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) < 0; } \ else { return len < xr.len; } } \ bool object_##NAME::operator> (const object_class* x) const { \ - const_raw xr(x->xraw()); \ + raw xr(x->xraw()); \ if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } \ else { return len > xr.len; } } \ void object_##NAME::pack(dynamic_packer& p) const \ { p.pack_raw(ptr, len); } \ const object_class* object_##NAME::inspect(std::ostream& s) const \ - { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape + { (s << '"').write(ptr, len) << '"'; return this; } // FIXME escape +// FIXME +RAW_OBJECT(mutable_raw_ref, + /*mutable_raw object_mutable_raw_ref::xraw() { return mutable_raw(ptr, len); }*/ + raw object_mutable_raw_ref::xraw() const { return raw(ptr, len); } ) + RAW_OBJECT(raw_ref, - raw object_raw_ref::xraw() { return raw(ptr, len); } - const_raw object_raw_ref::xraw() const { return const_raw(ptr, len); } ) - -RAW_OBJECT(const_raw_ref, - const_raw object_const_raw_ref::xraw() const { return const_raw(ptr, len); } ) + raw object_raw_ref::xraw() const { return raw(ptr, len); } ) #undef RAW_OBJECT(NAME, EXTRA) diff --git a/cpp/object.hpp b/cpp/object.hpp index 959d84e..9f91677 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -37,25 +37,25 @@ class positive_overflow_error : public overflow_error { }; class negative_overflow_error : public overflow_error { }; -struct raw { - explicit raw() : ptr(NULL), len(0) {} - explicit raw(void* p, size_t l) : ptr(p), len(l) {} +struct mutable_raw { + explicit mutable_raw() : ptr(NULL), len(0) {} + explicit mutable_raw(char* p, size_t l) : ptr(p), len(l) {} public: - void* ptr; + char* ptr; size_t len; public: - std::string str() { return std::string((const char*)ptr, len); } + std::string str() { return std::string(ptr, len); } }; -struct const_raw { - explicit const_raw() : ptr(NULL), len(0) {} - explicit const_raw(const void* p, size_t l) : ptr(p), len(l) {} - const_raw(const raw& m) : ptr(m.ptr), len(m.len) {} +struct raw { + explicit raw() : ptr(NULL), len(0) {} + explicit raw(const char* p, size_t l) : ptr(p), len(l) {} + raw(const mutable_raw& m) : ptr(m.ptr), len(m.len) {} public: - const void* ptr; + const char* ptr; size_t len; public: - std::string str() { return std::string((const char*)ptr, len); } + std::string str() { return std::string(ptr, len); } }; @@ -81,10 +81,10 @@ struct object_class { virtual int64_t xi64 () const { throw cast_error(); } virtual float xfloat () const { throw cast_error(); } virtual double xdouble() const { throw cast_error(); } - virtual raw xraw () { throw cast_error(); } - virtual array& xarray () { throw cast_error(); } - virtual map& xmap () { throw cast_error(); } - virtual const_raw xraw () const { throw cast_error(); } + //virtual mutable_raw xraw () { throw cast_error(); } // FIXME + virtual array& xarray() { throw cast_error(); } + virtual map& xmap () { throw cast_error(); } + virtual raw xraw () const { throw cast_error(); } virtual const array& xarray() const { throw cast_error(); } virtual const map& xmap () const { throw cast_error(); } virtual bool operator== (const object_class* x) const { return false; } @@ -103,10 +103,10 @@ struct object_class { operator int64_t() const { return xi64(); } operator float() const { return xfloat(); } operator double() const { return xdouble(); } - operator raw() { return xraw(); } - operator array&() { return xarray(); } - operator map&() { return xmap(); } - operator const_raw() const { return xraw(); } + //operator mutable_raw() { return xraw(); } // FIXME + operator array&() { return xarray(); } + operator map&() { return xmap(); } + operator raw() const { return xraw(); } operator const array&() const { return xarray(); } operator const map&() const { return xmap(); } virtual const object_class* inspect(std::ostream& s) const @@ -138,10 +138,10 @@ struct object { int64_t xi64 () const { return val->xi64(); } float xfloat () const { return val->xfloat(); } double xdouble() const { return val->xdouble(); } - raw xraw () { return val->xraw(); } - array& xarray () { return val->xarray(); } - map& xmap () { return val->xmap(); } - const_raw xraw () const { return const_cast(val)->xraw(); } + //mutable_raw xraw () { return val->xraw(); } // FIXME + array& xarray() { return val->xarray(); } + map& xmap () { return val->xmap(); } + raw xraw () const { return const_cast(val)->xraw(); } const array& xarray() const { return const_cast(val)->xarray(); } const map& xmap () const { return const_cast(val)->xmap(); } bool operator== (object x) const { return val->operator== (x.val); } @@ -149,6 +149,8 @@ struct object { bool operator< (object x) const { return val->operator< (x.val); } bool operator> (object x) const { return val->operator> (x.val); } void pack(dynamic_packer& p) const { val->pack(p); } + template + void pack(Stream& s) const { dynamic_packer p(s); pack(p); } operator bool() const { return val->operator bool(); } operator uint8_t() const { return val->operator uint8_t(); } operator uint16_t() const { return val->operator uint16_t(); } @@ -160,10 +162,10 @@ struct object { operator int64_t() const { return val->operator int64_t(); } operator float() const { return val->operator float(); } operator double() const { return val->operator double(); } - operator raw() { return val->operator raw(); } - operator array&() { return val->operator array&(); } - operator map&() { return val->operator map&(); } - operator const_raw() const { return val->operator const_raw(); } + //operator mutable_raw() { return val->operator mutable_raw(); } // FIXME + operator array&() { return val->operator array&(); } + operator map&() { return val->operator map&(); } + operator raw() const { return val->operator raw(); } operator const array&() const { return val->operator const array&(); } operator const map&() const { return val->operator const map&(); } const object& inspect(std::ostream& s) const @@ -275,8 +277,9 @@ private: \ uint32_t len; \ }; -RAW_CLASS(raw_ref, void*, raw xraw(); const_raw xraw() const; ) -RAW_CLASS(const_raw_ref, const void*, const_raw xraw() const; ) +// FIXME +RAW_CLASS(mutable_raw_ref, char*, /*mutable_raw xraw();*/ raw xraw() const; ) +RAW_CLASS(raw_ref, const char*, raw xraw() const; ) #undef RAW_CLASS(NAME, TYPE, EXTRA) diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 94b5b9a..9580679 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -50,7 +50,7 @@ public: void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } void pack_string(const char* b) { pack_string_impl(m_stream, b); } - void pack_raw(const void* b, size_t l) { pack_raw_impl(m_stream, b, l); } + void pack_raw(const char* b, size_t l) { pack_raw_impl(m_stream, (const void*)b, l); } private: static void pack_int_impl(Stream& x, int d); @@ -134,7 +134,7 @@ public: void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } void pack_string(const char* b) { pack_string_impl(m_stream, b); } - void pack_raw(const void* b, size_t l) { pack_raw_impl(m_stream, b, l); } + void pack_raw(const char* b, size_t l) { pack_raw_impl(m_stream, (const void*)b, l); } private: static void pack_int_impl(dynamic_stream& x, int d); diff --git a/cpp/test.cpp b/cpp/test.cpp index b63f9a1..68050a3 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -4,7 +4,7 @@ //#include #include #include -#include +#include class checker { public: @@ -176,7 +176,7 @@ int main(void) // 2. read data to buffer() up to buffer_capacity() bytes size_t sz = stream.readsome( - (char*)upk.buffer(), + upk.buffer(), upk.buffer_capacity()); total_bytes -= sz; @@ -195,7 +195,7 @@ int main(void) msgpack::object o = upk.data(); // 5.2. the parsed object is valid until the zone is deleted - boost::scoped_ptr pz(upk.release_zone()); + std::auto_ptr pz(upk.release_zone()); std::cout << o << std::endl; ++num_msg; diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index de33051..fe7f4b5 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -31,9 +31,9 @@ struct unpacker::context { ~context() { } - int execute(const void* data, size_t len, size_t* off) + int execute(const char* data, size_t len, size_t* off) { - return msgpack_unpacker_execute(&m_ctx, (const char*)data, len, off); + return msgpack_unpacker_execute(&m_ctx, data, len, off); } object_class* data() @@ -171,7 +171,7 @@ void unpacker::reset() } -object unpacker::unpack(const void* data, size_t len, zone& z) +object unpacker::unpack(const char* data, size_t len, zone& z) { context ctx(&z); size_t off = 0; diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index f5896f3..473e8e9 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -100,7 +100,7 @@ private: unpacker(const unpacker&); public: - static object unpack(const void* data, size_t len, zone& z); + static object unpack(const char* data, size_t len, zone& z); }; @@ -136,7 +136,7 @@ inline void unpacker::remove_nonparsed_buffer() { m_used = m_off; } -inline object unpack(const void* data, size_t len, zone& z) +inline object unpack(const char* data, size_t len, zone& z) { return unpacker::unpack(data, len, z); } diff --git a/cpp/unpack_inline.cpp b/cpp/unpack_inline.cpp index 37e0b66..40f2769 100644 --- a/cpp/unpack_inline.cpp +++ b/cpp/unpack_inline.cpp @@ -71,12 +71,12 @@ static inline void msgpack_unpack_array_item(zone** z, object_class* c, object_c { reinterpret_cast(c)->push_back(o); } static inline object_class* msgpack_unpack_map_start(zone** z, unsigned int n) -{ return (*z)->narray(); } +{ return (*z)->nmap(); } static inline void msgpack_unpack_map_item(zone** z, object_class* c, object_class* k, object_class* v) { reinterpret_cast(c)->store(k, v); } -static inline object_class* msgpack_unpack_raw(zone** z, const void* b, const void* p, size_t l) +static inline object_class* msgpack_unpack_raw(zone** z, const char* b, const char* p, unsigned int l) { return (*z)->nraw_ref(p, l); } diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index d63fae8..8af8da2 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -58,19 +58,20 @@ public: object_float* nfloat (float v) { return new (alloc()) object_float(v); } object_double* ndouble(double v) { return new (alloc()) object_double(v); } - object_raw_ref* nraw_ref(void* ptr, uint32_t len) + + object_mutable_raw_ref* nraw_ref(char* ptr, uint32_t len) + { return new (alloc()) object_mutable_raw_ref(ptr, len); } + + object_raw_ref* nraw_ref(const char* ptr, uint32_t len) { return new (alloc()) object_raw_ref(ptr, len); } - object_const_raw_ref* nraw_ref(const void* ptr, uint32_t len) - { return new (alloc()) object_const_raw_ref(ptr, len); } - - object_raw_ref* nraw_copy(const void* ptr, uint32_t len) + object_mutable_raw_ref* nraw_copy(const char* ptr, uint32_t len) { - void* copy = malloc(len); + char* copy = (char*)malloc(len); if(!copy) { throw std::bad_alloc(); } - object_raw_ref* o; + object_mutable_raw_ref* o; try { - o = new (alloc()) object_raw_ref(copy, len); + o = new (alloc()) object_mutable_raw_ref(copy, len); push_finalizer(&zone::finalize_free, NULL, copy); } catch (...) { free(copy); @@ -80,6 +81,17 @@ public: return o; } + + object_mutable_raw_ref* nraw_cstr_ref(char* str) + { return nraw_ref(str, strlen(str)); } + + object_raw_ref* nraw_cstr_ref(const char* str) + { return nraw_ref(str, strlen(str)); } + + object_mutable_raw_ref* nraw_cstr_copy(const char* str) + { return nraw_copy(str, strlen(str)); } + + object_array* narray() { return new (alloc()) object_array(); } diff --git a/msgpack/unpack/callback.h b/msgpack/unpack/callback.h index 51508cc..315bb9e 100644 --- a/msgpack/unpack/callback.h +++ b/msgpack/unpack/callback.h @@ -20,5 +20,5 @@ msgpack_object msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned in msgpack_object msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n); void msgpack_unpack_map_item(msgpack_unpack_context* x, msgpack_object c, msgpack_object k, msgpack_object v); msgpack_object msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l); -msgpack_object msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, const void* p, size_t l); +msgpack_object msgpack_unpack_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l); diff --git a/msgpack/unpack/inline_context.h b/msgpack/unpack/inline_context.h index 22698d5..e764c09 100644 --- a/msgpack/unpack/inline_context.h +++ b/msgpack/unpack/inline_context.h @@ -42,7 +42,7 @@ typedef struct { typedef struct { msgpack_unpack_context user; // must be first unsigned int cs; - size_t trail; + unsigned int trail; unsigned int top; msgpack_unpacker_stack stack[MSG_STACK_SIZE]; } msgpack_unpacker; diff --git a/msgpack/unpack/inline_impl.h b/msgpack/unpack/inline_impl.h index fdbe7f6..443a40c 100644 --- a/msgpack/unpack/inline_impl.h +++ b/msgpack/unpack/inline_impl.h @@ -168,7 +168,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len const unsigned char* const pe = (unsigned char*)data + len; const void* n = NULL; - size_t trail = ctx->trail; + unsigned int trail = ctx->trail; unsigned int cs = ctx->cs; unsigned int top = ctx->top; msgpack_unpacker_stack* stack = ctx->stack; @@ -187,7 +187,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len /*printf("obj %d\n",obj);*/ \ goto _push #define push_variable_value(func, base, pos, len) \ - obj = func(user, (const void*)base, (const void*)pos, len); \ + obj = func(user, (const char*)base, (const char*)pos, len); \ /*printf("obj %d\n",obj);*/ \ goto _push diff --git a/ruby/unpack_inline.c b/ruby/unpack_inline.c index b30754b..849441c 100644 --- a/ruby/unpack_inline.c +++ b/ruby/unpack_inline.c @@ -71,7 +71,7 @@ static inline VALUE msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned static inline void msgpack_unpack_map_item(msgpack_unpack_context* x, VALUE c, VALUE k, VALUE v) { rb_hash_aset(c, k, v); } -static inline VALUE msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, const void* p, size_t l) +static inline VALUE msgpack_unpack_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l) { return rb_str_new(p, l); } #include "msgpack/unpack/inline_impl.h" From 48e0274505583affa8546cdffe19761033622aa2 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:57 +0000 Subject: [PATCH 0017/1172] lang/c/msgpack: fixed packaging problem git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@64 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/configure.in | 2 +- c/pack_inline.h | 2 +- c/unpack.c | 2 +- c/unpack_context.h | 2 +- cpp/Makefile.am | 3 ++- cpp/configure.in | 2 +- cpp/object.hpp | 6 +++--- cpp/zone.cpp | 2 +- msgpack/pack/inline_impl.h | 10 +++++----- msgpack/unpack/inline_impl.h | 15 ++++++++------- 10 files changed, 24 insertions(+), 22 deletions(-) diff --git a/c/configure.in b/c/configure.in index 4bc7e3e..1cacbc4 100644 --- a/c/configure.in +++ b/c/configure.in @@ -1,4 +1,4 @@ -AC_INIT(pack.h) +AC_INIT(pack.c) AM_INIT_AUTOMAKE(msgpackc, 0.1.0) AC_CONFIG_HEADER(config.h) diff --git a/c/pack_inline.h b/c/pack_inline.h index acd96fa..dd43a20 100644 --- a/c/pack_inline.h +++ b/c/pack_inline.h @@ -18,7 +18,7 @@ #ifndef PACK_INLINE_H__ #define PACK_INLINE_H__ -#include "pack.h" +#include "msgpack/pack.h" typedef msgpack_pack_t* msgpack_pack_context; diff --git a/c/unpack.c b/c/unpack.c index 012888e..a2fc066 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "unpack.h" +#include "msgpack/unpack.h" #include "unpack_context.h" #include diff --git a/c/unpack_context.h b/c/unpack_context.h index 7337c9e..d7b0388 100644 --- a/c/unpack_context.h +++ b/c/unpack_context.h @@ -18,7 +18,7 @@ #ifndef UNPACK_CONTEXT_H__ #define UNPACK_CONTEXT_H__ -#include "unpack.h" +#include "msgpack/unpack.h" typedef void* msgpack_object; diff --git a/cpp/Makefile.am b/cpp/Makefile.am index b9ecb1d..d1fa826 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -14,7 +14,8 @@ nobase_include_HEADERS = \ msgpack/zone.hpp noinst_HEADERS = \ - unpack_context.hpp + unpack_context.hpp \ + msgpack/zone.hpp.erb msgpack/zone.hpp: msgpack/zone.hpp.erb erb $< > $@ diff --git a/cpp/configure.in b/cpp/configure.in index e9e21ea..5126be4 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -1,4 +1,4 @@ -AC_INIT(object.hpp) +AC_INIT(object.cpp) AM_INIT_AUTOMAKE(msgpack, 0.1.0) AC_CONFIG_HEADER(config.h) diff --git a/cpp/object.hpp b/cpp/object.hpp index 9f91677..456210c 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -232,7 +232,7 @@ INTEGER_CLASS(int16_t, i16) INTEGER_CLASS(int32_t, i32) INTEGER_CLASS(int64_t, i64) -#undef INTEGER_CLASS(TYPE, NAME) +#undef INTEGER_CLASS #define FLOAT_CLASS(TYPE, NAME) \ @@ -260,7 +260,7 @@ private: \ FLOAT_CLASS(float, float) FLOAT_CLASS(double, double) -#undef FLOAT_CLASS(TYPE, NAME) +#undef FLOAT_CLASS #define RAW_CLASS(NAME, TYPE, EXTRA) \ @@ -281,7 +281,7 @@ private: \ RAW_CLASS(mutable_raw_ref, char*, /*mutable_raw xraw();*/ raw xraw() const; ) RAW_CLASS(raw_ref, const char*, raw xraw() const; ) -#undef RAW_CLASS(NAME, TYPE, EXTRA) +#undef RAW_CLASS struct object_array : object_class, object_container_mixin { diff --git a/cpp/zone.cpp b/cpp/zone.cpp index ff22fc3..4cc50d7 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -#include "zone.hpp" +#include "msgpack/zone.hpp" namespace msgpack { diff --git a/msgpack/pack/inline_impl.h b/msgpack/pack/inline_impl.h index d4d5a5a..635b697 100644 --- a/msgpack/pack/inline_impl.h +++ b/msgpack/pack/inline_impl.h @@ -52,7 +52,7 @@ #endif -#ifndef msgpack_pack_inline_func(name) +#ifndef msgpack_pack_inline_func #define msgpack_pack_inline_func(name) \ inline void msgpack_pack_##name #endif @@ -293,11 +293,11 @@ msgpack_pack_inline_func(raw)(msgpack_pack_context x, const void* b, size_t l) } -#undef msgpack_pack_inline_func(name) +#undef msgpack_pack_inline_func -#undef STORE_BE16(d) -#undef STORE_BE32(d) -#undef STORE_BE64(d) +#undef STORE_BE16 +#undef STORE_BE32 +#undef STORE_BE64 #endif /* msgpack/pack/inline_impl.h */ diff --git a/msgpack/unpack/inline_impl.h b/msgpack/unpack/inline_impl.h index 443a40c..a6557f9 100644 --- a/msgpack/unpack/inline_impl.h +++ b/msgpack/unpack/inline_impl.h @@ -175,6 +175,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len msgpack_unpack_context* user = &ctx->user; msgpack_object obj; + msgpack_unpacker_stack* c = NULL; int ret; @@ -381,7 +382,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len _push: if(top == 0) { goto _finish; } - msgpack_unpacker_stack* c = &stack[top-1]; + c = &stack[top-1]; switch(c->ct) { case CT_ARRAY_ITEM: msgpack_unpack_array_item(user, c->obj, obj); @@ -444,16 +445,16 @@ _end: } -#ifdef betoh16(x) -#undef betoh16(x) +#ifdef betoh16 +#undef betoh16 #endif -#ifdef betoh32(x) -#undef betoh32(x) +#ifdef betoh32 +#undef betoh32 #endif -#ifdef betoh64(x) -#undef betoh64(x) +#ifdef betoh64 +#undef betoh64 #endif #ifdef __cplusplus From 76dda6d36e5a2edbe21443bd7344c41160373e2e Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:57 +0000 Subject: [PATCH 0018/1172] lang/c/msgpack: fixed cpp/Makefile.am git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@65 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/Makefile.am | 6 ++++++ cpp/object.cpp | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index d1fa826..08ea21f 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -17,6 +17,12 @@ noinst_HEADERS = \ unpack_context.hpp \ msgpack/zone.hpp.erb +# FIXME +object.lo: msgpack/zone.hpp +unpack.lo: msgpack/zone.hpp +unpack_context.lo: msgpack/zone.hpp +zone.lo: msgpack/zone.hpp + msgpack/zone.hpp: msgpack/zone.hpp.erb erb $< > $@ diff --git a/cpp/object.cpp b/cpp/object.cpp index 4068007..7bb9841 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -258,7 +258,7 @@ INTEGER_OBJECT(i16) INTEGER_OBJECT(i32) INTEGER_OBJECT(i64) -#undef INTEGER_OBJECT(NAME) +#undef INTEGER_OBJECT #define FLOAT_OBJECT(NAME) \ @@ -313,7 +313,7 @@ const object_class* object_##NAME::inspect(std::ostream& s) const \ FLOAT_OBJECT(float) FLOAT_OBJECT(double) -#undef FLOAT_OBJECT(NAME) +#undef FLOAT_OBJECT #define RAW_OBJECT(NAME, EXTRA) \ @@ -345,7 +345,7 @@ RAW_OBJECT(mutable_raw_ref, RAW_OBJECT(raw_ref, raw object_raw_ref::xraw() const { return raw(ptr, len); } ) -#undef RAW_OBJECT(NAME, EXTRA) +#undef RAW_OBJECT array& object_array::xarray() { return val; } From 1222466a1c52161a3da3c3c5ce552d4b90e32bf6 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:57 +0000 Subject: [PATCH 0019/1172] lang/c/msgpack: c-macro based template git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@66 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- Makefile.am | 6 +- c/Makefile.am | 7 +- c/bench.c | 86 ++-- c/bench.mk | 9 + c/bench_inline.c | 325 ------------ c/pack.c | 21 +- c/pack.h | 33 +- c/pack_inline.h | 32 -- c/unpack.c | 112 ++++- c/unpack.h | 35 +- c/unpack_context.h | 30 -- c/unpack_inline.c | 79 --- cpp/Makefile.am | 3 - cpp/bench.cpp | 10 +- cpp/bench.mk | 9 + cpp/object.cpp | 16 +- cpp/pack.hpp | 80 ++- cpp/unpack.cpp | 116 ++++- cpp/unpack.hpp | 6 +- cpp/unpack_context.hpp | 31 -- cpp/unpack_inline.cpp | 86 ---- .../{pack/inline_impl.h => pack_template.h} | 63 +-- msgpack/unpack/callback.h | 24 - msgpack/unpack/inline_context.h | 59 --- msgpack/unpack/inline_impl.h | 465 ------------------ msgpack/unpack_define.h | 127 +++++ msgpack/unpack_template.h | 360 ++++++++++++++ ruby/gem/Manifest.txt | 9 +- ruby/gem/lib/msgpack/version.rb | 2 +- ruby/gengem.sh | 9 +- ruby/pack.c | 40 +- ruby/pack_inline.h | 35 -- ruby/unpack.c | 103 +++- ruby/unpack_context.h | 35 -- ruby/unpack_inline.c | 78 --- 35 files changed, 1034 insertions(+), 1507 deletions(-) create mode 100644 c/bench.mk delete mode 100644 c/bench_inline.c delete mode 100644 c/pack_inline.h delete mode 100644 c/unpack_context.h delete mode 100644 c/unpack_inline.c create mode 100644 cpp/bench.mk delete mode 100644 cpp/unpack_context.hpp delete mode 100644 cpp/unpack_inline.cpp rename msgpack/{pack/inline_impl.h => pack_template.h} (77%) delete mode 100644 msgpack/unpack/callback.h delete mode 100644 msgpack/unpack/inline_context.h delete mode 100644 msgpack/unpack/inline_impl.h create mode 100644 msgpack/unpack_define.h create mode 100644 msgpack/unpack_template.h delete mode 100644 ruby/pack_inline.h delete mode 100644 ruby/unpack_context.h delete mode 100644 ruby/unpack_inline.c diff --git a/Makefile.am b/Makefile.am index 9d1282d..d79f7eb 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,7 +1,7 @@ SUBDIRS = c cpp nobase_include_HEADERS = \ - msgpack/pack/inline_impl.h \ - msgpack/unpack/inline_context.h \ - msgpack/unpack/inline_impl.h + msgpack/pack_template.h \ + msgpack/unpack_define.h \ + msgpack/unpack_template.h diff --git a/c/Makefile.am b/c/Makefile.am index af8f3ad..f1c57e9 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -2,17 +2,12 @@ lib_LTLIBRARIES = libmsgpackc.la libmsgpackc_la_SOURCES = \ pack.c \ - unpack.c \ - unpack_inline.c + unpack.c nobase_include_HEADERS = \ msgpack.h \ msgpack/pack.h \ msgpack/unpack.h -noinst_HEADERS = \ - pack_inline.h \ - unpack_context.h - libmsgpackc_la_LDFLAGS = -version-info 0:0:0 diff --git a/c/bench.c b/c/bench.c index 6b8466f..fa717c0 100644 --- a/c/bench.c +++ b/c/bench.c @@ -16,14 +16,15 @@ void reset_timer() gettimeofday(&g_timer, NULL); } -double show_timer() +void show_timer(size_t bufsz) { struct timeval endtime; gettimeofday(&endtime, NULL); double sec = (endtime.tv_sec - g_timer.tv_sec) + (double)(endtime.tv_usec - g_timer.tv_usec) / 1000 / 1000; printf("%f sec\n", sec); - return sec; + printf("%f MB\n", ((double)bufsz)/1024/1024); + printf("%f Mbps\n", ((double)bufsz)*8/sec/1000/1000); } @@ -38,32 +39,33 @@ static int reformat_start_array(void * ctx) { return 1; } static int reformat_end_array(void * ctx) { return 1; } -static void* unpack_unsigned_int_8(void* data, uint8_t d) { return NULL; } -static void* unpack_unsigned_int_16(void* data, uint16_t d) { return NULL; } -static void* unpack_unsigned_int_32(void* data, uint32_t d) { return NULL; } -static void* unpack_unsigned_int_64(void* data, uint64_t d) { return NULL; } -static void* unpack_signed_int_8(void* data, int8_t d) { return NULL; } -static void* unpack_signed_int_16(void* data, int16_t d) { return NULL; } -static void* unpack_signed_int_32(void* data, int32_t d) { return NULL; } -static void* unpack_signed_int_64(void* data, int64_t d) { return NULL; } +static void* unpack_uint8(void* data, uint8_t d) { return NULL; } +static void* unpack_uint16(void* data, uint16_t d) { return NULL; } +static void* unpack_uint32(void* data, uint32_t d) { return NULL; } +static void* unpack_uint64(void* data, uint64_t d) { return NULL; } +static void* unpack_int8(void* data, int8_t d) { return NULL; } +static void* unpack_int16(void* data, int16_t d) { return NULL; } +static void* unpack_int32(void* data, int32_t d) { return NULL; } +static void* unpack_int64(void* data, int64_t d) { return NULL; } static void* unpack_float(void* data, float d) { return NULL; } static void* unpack_double(void* data, double d) { return NULL; } static void* unpack_nil(void* data) { return NULL; } static void* unpack_true(void* data) { return NULL; } static void* unpack_false(void* data) { return NULL; } -static void* unpack_array_start(void* data, unsigned int n) { return NULL; } +static void* unpack_array(void* data, unsigned int n) { return NULL; } static void unpack_array_item(void* data, void* c, void* o) { } -static void* unpack_map_start(void* data, unsigned int n) { return NULL; } +static void* unpack_map(void* data, unsigned int n) { return NULL; } static void unpack_map_item(void* data, void* c, void* k, void* v) { } static void* unpack_raw(void* data, const char* b, const char* p, unsigned int l) { /*printf("unpack raw %p %lu\n",p,l);*/ return NULL; } + typedef struct { size_t allocated; size_t length; char* buffer; } pack_buffer; -static const size_t PACK_INITIAL_BUFFER_SIZE = 512; +static const size_t PACK_INITIAL_BUFFER_SIZE = 32*1024; static void pack_buffer_init(pack_buffer* data) { @@ -84,7 +86,7 @@ static void pack_buffer_free(pack_buffer* data) free(data->buffer); } -static void pack_append_buffer(void* user, const unsigned char* b, unsigned int l) +static void pack_append_buffer(void* user, const char* b, unsigned int l) { pack_buffer* data = (pack_buffer*)user; if(data->allocated - data->length < l) { @@ -128,7 +130,6 @@ void bench_json(void) yajl_handle h = yajl_alloc(&callbacks, &hcfg, NULL); - double sec; const unsigned char * buf; unsigned int len; @@ -143,11 +144,9 @@ void bench_json(void) } yajl_gen_array_close(g); } - sec = show_timer(); + show_timer(len); yajl_gen_get_buf(g, &buf, &len); - printf("%u KB\n", len / 1024); - printf("%f MB/s\n", len / sec / 1024 / 1024); puts("----"); puts("parse integer"); @@ -159,9 +158,7 @@ void bench_json(void) fprintf(stderr, (const char *) str); } } - sec = show_timer(); - - printf("%f MB/s\n", len / sec / 1024 / 1024); + show_timer(len); //yajl_gen_clear(g); @@ -182,11 +179,9 @@ void bench_json(void) } yajl_gen_array_close(g); } - sec = show_timer(); + show_timer(len); yajl_gen_get_buf(g, &buf, &len); - printf("%u KB\n", len / 1024); - printf("%f MB/s\n", len / sec / 1024 / 1024); puts("----"); puts("parse string"); @@ -198,15 +193,14 @@ void bench_json(void) fprintf(stderr, (const char *) str); } } - sec = show_timer(); - - printf("%f MB/s\n", len / sec / 1024 / 1024); + show_timer(len); yajl_gen_free(g); yajl_free(h); } + void bench_msgpack(void) { puts("== MessagePack =="); @@ -214,33 +208,33 @@ void bench_msgpack(void) pack_buffer mpkbuf; pack_buffer_init(&mpkbuf); + msgpack_pack_t* mpk = msgpack_pack_new( &mpkbuf, pack_append_buffer); msgpack_unpack_callback cb = { - unpack_unsigned_int_8, - unpack_unsigned_int_16, - unpack_unsigned_int_32, - unpack_unsigned_int_64, - unpack_signed_int_8, - unpack_signed_int_16, - unpack_signed_int_32, - unpack_signed_int_64, + unpack_uint8, + unpack_uint16, + unpack_uint32, + unpack_uint64, + unpack_int8, + unpack_int16, + unpack_int32, + unpack_int64, unpack_float, unpack_double, unpack_nil, unpack_true, unpack_false, - unpack_array_start, + unpack_array, unpack_array_item, - unpack_map_start, + unpack_map, unpack_map_item, unpack_raw, }; msgpack_unpack_t* mupk = msgpack_unpack_new(NULL, &cb); - double sec; size_t len; const char* buf; @@ -254,12 +248,10 @@ void bench_msgpack(void) msgpack_pack_unsigned_int(mpk, i); } } - sec = show_timer(); + show_timer(mpkbuf.length); len = mpkbuf.length; buf = mpkbuf.buffer; - printf("%lu KB\n", len / 1024); - printf("%f MB/s\n", len / sec / 1024 / 1024); puts("----"); puts("unpack integer"); @@ -273,9 +265,7 @@ void bench_msgpack(void) fprintf(stderr, "Not finished.\n"); } } - sec = show_timer(); - - printf("%f MB/s\n", len / sec / 1024 / 1024); + show_timer(mpkbuf.length); pack_buffer_reset(&mpkbuf); @@ -292,12 +282,10 @@ void bench_msgpack(void) msgpack_pack_raw(mpk, TASK_STR_PTR, i); } } - sec = show_timer(); + show_timer(mpkbuf.length); len = mpkbuf.length; buf = mpkbuf.buffer; - printf("%lu KB\n", len / 1024); - printf("%f MB/s\n", len / sec / 1024 / 1024); puts("----"); puts("unpack string"); @@ -311,9 +299,7 @@ void bench_msgpack(void) fprintf(stderr, "Not finished.\n"); } } - sec = show_timer(); - - printf("%f MB/s\n", len / sec / 1024 / 1024); + show_timer(mpkbuf.length); msgpack_unpack_free(mupk); diff --git a/c/bench.mk b/c/bench.mk new file mode 100644 index 0000000..c765e31 --- /dev/null +++ b/c/bench.mk @@ -0,0 +1,9 @@ + +CFLAGS += -Wall -g -I. -I.. -O4 +LDFLAGS += -lyajl + +all: bench + +bench: bench.o pack.o unpack.o pack.h unpack.h + $(CC) bench.o pack.o unpack.o $(CFLAGS) $(LDFLAGS) -o $@ + diff --git a/c/bench_inline.c b/c/bench_inline.c deleted file mode 100644 index bb3c431..0000000 --- a/c/bench_inline.c +++ /dev/null @@ -1,325 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include -#include - - -static struct timeval g_timer; - -void reset_timer() -{ - gettimeofday(&g_timer, NULL); -} - -double show_timer() -{ - struct timeval endtime; - gettimeofday(&endtime, NULL); - double sec = (endtime.tv_sec - g_timer.tv_sec) - + (double)(endtime.tv_usec - g_timer.tv_usec) / 1000 / 1000; - printf("%f sec\n", sec); - return sec; -} - - -static int reformat_null(void * ctx) { return 1; } -static int reformat_boolean(void * ctx, int boolean) { return 1; } -static int reformat_number(void * ctx, const char * s, unsigned int l) { return 1; } -static int reformat_string(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } -static int reformat_map_key(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } -static int reformat_start_map(void * ctx) { return 1; } -static int reformat_end_map(void * ctx) { return 1; } -static int reformat_start_array(void * ctx) { return 1; } -static int reformat_end_array(void * ctx) { return 1; } - - -typedef void* msgpack_object; - -typedef struct { -} msgpack_unpack_context; - -#include "msgpack/unpack/inline_context.h" - -static inline void* msgpack_unpack_init(msgpack_unpack_context* x) { return NULL; } -static inline void* msgpack_unpack_unsigned_int_8(msgpack_unpack_context* x, uint8_t d) { return NULL; } -static inline void* msgpack_unpack_unsigned_int_16(msgpack_unpack_context* x, uint16_t d) { return NULL; } -static inline void* msgpack_unpack_unsigned_int_32(msgpack_unpack_context* x, uint32_t d) { return NULL; } -static inline void* msgpack_unpack_unsigned_int_64(msgpack_unpack_context* x, uint64_t d) { return NULL; } -static inline void* msgpack_unpack_signed_int_8(msgpack_unpack_context* x, int8_t d) { return NULL; } -static inline void* msgpack_unpack_signed_int_16(msgpack_unpack_context* x, int16_t d) { return NULL; } -static inline void* msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int32_t d) { return NULL; } -static inline void* msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d) { return NULL; } -static inline void* msgpack_unpack_float(msgpack_unpack_context* x, float d) { return NULL; } -static inline void* msgpack_unpack_double(msgpack_unpack_context* x, double d) { return NULL; } -static inline void* msgpack_unpack_nil(msgpack_unpack_context* x) { return NULL; } -static inline void* msgpack_unpack_true(msgpack_unpack_context* x) { return NULL; } -static inline void* msgpack_unpack_false(msgpack_unpack_context* x) { return NULL; } -static inline void* msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned int n) { return NULL; } -static inline void msgpack_unpack_array_item(msgpack_unpack_context* x, void* c, void* o) { } -static inline void* msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n) { return NULL; } -static inline void msgpack_unpack_map_item(msgpack_unpack_context* x, void* c, void* k, void* v) { } -static inline void* msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, const void* p, size_t l) { return NULL; } - -#include "msgpack/unpack/inline_impl.h" - -typedef struct { - size_t allocated; - size_t length; - char* buffer; -} pack_buffer; - -static const size_t PACK_INITIAL_BUFFER_SIZE = 512; - -static void pack_buffer_init(pack_buffer* data) -{ - data->buffer = malloc(PACK_INITIAL_BUFFER_SIZE); - data->length = 0; - data->allocated = PACK_INITIAL_BUFFER_SIZE; -} - -static void pack_buffer_reset(pack_buffer* data) -{ - data->buffer = realloc(data->buffer, PACK_INITIAL_BUFFER_SIZE); - data->allocated = PACK_INITIAL_BUFFER_SIZE; - data->length = 0; -} - -static void pack_buffer_free(pack_buffer* data) -{ - free(data->buffer); -} - -static void pack_append_buffer(void* user, const unsigned char* b, unsigned int l) -{ - pack_buffer* data = (pack_buffer*)user; - if(data->allocated - data->length < l) { - data->buffer = realloc(data->buffer, data->allocated*2); - data->allocated *= 2; - } - memcpy(data->buffer + data->length, b, l); - data->length += l; -} - - -static const unsigned int TASK_INT_NUM = 1<<24; -static const unsigned int TASK_STR_LEN = 1<<15; -//static const unsigned int TASK_INT_NUM = 1<<20; -//static const unsigned int TASK_STR_LEN = 1<<12; -static const char* TASK_STR_PTR; - - -void bench_json(void) -{ - puts("== JSON =="); - - - yajl_gen_config gcfg = {0, NULL}; - yajl_gen g = yajl_gen_alloc(&gcfg); - - yajl_parser_config hcfg = { 0, 0 }; - yajl_callbacks callbacks = { - reformat_null, - reformat_boolean, - NULL, - NULL, - reformat_number, - reformat_start_map, - reformat_map_key, - reformat_end_map, - reformat_start_array, - reformat_end_array - }; - yajl_handle h = yajl_alloc(&callbacks, &hcfg, NULL); - - - double sec; - const unsigned char * buf; - unsigned int len; - - - puts("generate integer"); - reset_timer(); - { - unsigned int i; - yajl_gen_array_open(g); - for(i=0; i < TASK_INT_NUM; ++i) { - yajl_gen_integer(g, i); - } - yajl_gen_array_close(g); - } - sec = show_timer(); - - yajl_gen_get_buf(g, &buf, &len); - printf("%u KB\n", len / 1024); - printf("%f Mbps\n", len / sec / 1024 / 1024); - - puts("----"); - puts("parse integer"); - reset_timer(); - { - yajl_status stat = yajl_parse(h, buf, len); - if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { - unsigned char * str = yajl_get_error(h, 1, buf, len); - fprintf(stderr, (const char *) str); - } - } - sec = show_timer(); - - printf("%f Mbps\n", len / sec / 1024 / 1024); - - - //yajl_gen_clear(g); - yajl_gen_free(g); - g = yajl_gen_alloc(&gcfg); - yajl_free(h); - h = yajl_alloc(&callbacks, &hcfg, NULL); - - - puts("----"); - puts("generate string"); - reset_timer(); - { - unsigned int i; - yajl_gen_array_open(g); - for(i=0; i < TASK_STR_LEN; ++i) { - yajl_gen_string(g, (const unsigned char*)TASK_STR_PTR, i); - } - yajl_gen_array_close(g); - } - sec = show_timer(); - - yajl_gen_get_buf(g, &buf, &len); - printf("%u KB\n", len / 1024); - printf("%f Mbps\n", len / sec / 1024 / 1024); - - puts("----"); - puts("parse string"); - reset_timer(); - { - yajl_status stat = yajl_parse(h, buf, len); - if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { - unsigned char * str = yajl_get_error(h, 1, buf, len); - fprintf(stderr, (const char *) str); - } - } - sec = show_timer(); - - printf("%f Mbps\n", len / sec / 1024 / 1024); - - - yajl_gen_free(g); - yajl_free(h); -} - -void bench_msgpack(void) -{ - puts("== MessagePack =="); - - - pack_buffer mpkbuf; - pack_buffer_init(&mpkbuf); - msgpack_pack_t* mpk = msgpack_pack_new( - &mpkbuf, pack_append_buffer); - - msgpack_unpacker mupk; - msgpack_unpacker_init(&mupk); - - double sec; - size_t len; - const char* buf; - - - puts("pack integer"); - reset_timer(); - { - unsigned int i; - msgpack_pack_array(mpk, TASK_INT_NUM); - for(i=0; i < TASK_INT_NUM; ++i) { - msgpack_pack_unsigned_int(mpk, i); - } - } - sec = show_timer(); - - len = mpkbuf.length; - buf = mpkbuf.buffer; - printf("%lu KB\n", len / 1024); - printf("%f Mbps\n", len / sec / 1024 / 1024); - - puts("----"); - puts("unpack integer"); - reset_timer(); - { - size_t off = 0; - int ret = msgpack_unpacker_execute(&mupk, buf, len, &off); - if(ret < 0) { - fprintf(stderr, "Parse error.\n"); - } else if(ret == 0) { - fprintf(stderr, "Not finished.\n"); - } - } - sec = show_timer(); - - printf("%f Mbps\n", len / sec / 1024 / 1024); - - - pack_buffer_reset(&mpkbuf); - msgpack_unpacker_init(&mupk); - - - puts("----"); - puts("pack string"); - reset_timer(); - { - unsigned int i; - msgpack_pack_array(mpk, TASK_STR_LEN); - for(i=0; i < TASK_STR_LEN; ++i) { - msgpack_pack_raw(mpk, TASK_STR_PTR, i); - } - } - sec = show_timer(); - - len = mpkbuf.length; - buf = mpkbuf.buffer; - printf("%lu KB\n", len / 1024); - printf("%f Mbps\n", len / sec / 1024 / 1024); - - puts("----"); - puts("unpack string"); - reset_timer(); - { - size_t off = 0; - int ret = msgpack_unpacker_execute(&mupk, buf, len, &off); - if(ret < 0) { - fprintf(stderr, "Parse error.\n"); - } else if(ret == 0) { - fprintf(stderr, "Not finished.\n"); - } - } - sec = show_timer(); - - printf("%f Mbps\n", len / sec / 1024 / 1024); - - - msgpack_pack_free(mpk); - pack_buffer_free(&mpkbuf); -} - -int main(int argc, char* argv[]) -{ - char* str = malloc(TASK_STR_LEN); - memset(str, 'a', TASK_STR_LEN); - TASK_STR_PTR = str; - - bench_msgpack(); - //bench_json(); - - return 0; -} - - - diff --git a/c/pack.c b/c/pack.c index 05bd38d..766a9d1 100644 --- a/c/pack.c +++ b/c/pack.c @@ -15,11 +15,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "pack_inline.h" +#include "msgpack/pack.h" #include -#include -msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_callback_t callback) + +#define msgpack_pack_inline_func(name) \ + void msgpack_pack_##name + +#define msgpack_pack_user msgpack_pack_t* + +#define msgpack_pack_append_buffer(user, buf, len) \ + (*(user)->callback)((user)->data, (const char*)buf, len) + +#include "msgpack/pack_template.h" + +msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_append_buffer_t callback) { msgpack_pack_t* ctx = calloc(1, sizeof(msgpack_pack_t)); if(!ctx) { return NULL; } @@ -33,8 +43,3 @@ void msgpack_pack_free(msgpack_pack_t* ctx) free(ctx); } -static inline void msgpack_pack_append_buffer(msgpack_pack_t* ctx, const unsigned char* b, unsigned int l) -{ - ctx->callback(ctx->data, b, l); -} - diff --git a/c/pack.h b/c/pack.h index 3144f37..c6cadf4 100644 --- a/c/pack.h +++ b/c/pack.h @@ -21,26 +21,32 @@ #include #include -typedef void (*msgpack_pack_callback_t)(void* data, const unsigned char* b, unsigned int i); +#ifdef __cplusplus +extern "C" { +#endif + + +typedef void (*msgpack_pack_append_buffer_t)(void* data, const char* b, unsigned int i); typedef struct { void* data; - msgpack_pack_callback_t callback; + msgpack_pack_append_buffer_t callback; } msgpack_pack_t; -msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_callback_t callback); +msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_append_buffer_t callback); + void msgpack_pack_free(msgpack_pack_t* ctx); void msgpack_pack_int(msgpack_pack_t* ctx, int d); void msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); -void msgpack_pack_unsigned_int_8(msgpack_pack_t* ctx, uint8_t d); -void msgpack_pack_unsigned_int_16(msgpack_pack_t* ctx, uint16_t d); -void msgpack_pack_unsigned_int_32(msgpack_pack_t* ctx, uint32_t d); -void msgpack_pack_unsigned_int_64(msgpack_pack_t* ctx, uint64_t d); -void msgpack_pack_signed_int_8(msgpack_pack_t* ctx, int8_t d); -void msgpack_pack_signed_int_16(msgpack_pack_t* ctx, int16_t d); -void msgpack_pack_signed_int_32(msgpack_pack_t* ctx, int32_t d); -void msgpack_pack_signed_int_64(msgpack_pack_t* ctx, int64_t d); +void msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); +void msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); +void msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); +void msgpack_pack_uint64(msgpack_pack_t* ctx, uint64_t d); +void msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); +void msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); +void msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); +void msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); void msgpack_pack_float(msgpack_pack_t* ctx, float d); void msgpack_pack_double(msgpack_pack_t* ctx, double d); void msgpack_pack_nil(msgpack_pack_t* ctx); @@ -51,5 +57,10 @@ void msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); void msgpack_pack_string(msgpack_pack_t* ctx, const char* b); void msgpack_pack_raw(msgpack_pack_t* ctx, const void* b, size_t l); + +#ifdef __cplusplus +} +#endif + #endif /* msgpack/pack.h */ diff --git a/c/pack_inline.h b/c/pack_inline.h deleted file mode 100644 index dd43a20..0000000 --- a/c/pack_inline.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * MessagePack packing routine for C - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef PACK_INLINE_H__ -#define PACK_INLINE_H__ - -#include "msgpack/pack.h" - -typedef msgpack_pack_t* msgpack_pack_context; - -static inline void msgpack_pack_append_buffer(msgpack_pack_t* x, const unsigned char* b, unsigned int l); - -#include -#include /* __BYTE_ORDER */ -#include "msgpack/pack/inline_impl.h" - -#endif /* pack_inline.h */ - diff --git a/c/unpack.c b/c/unpack.c index a2fc066..3058427 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -16,15 +16,101 @@ * limitations under the License. */ #include "msgpack/unpack.h" -#include "unpack_context.h" +#include "msgpack/unpack_define.h" #include + +#define msgpack_unpack_struct(name) \ + struct template_##name + +#define msgpack_unpack_func(ret, name) \ + ret template_func_##name + +#define msgpack_unpack_callback(name) \ + template_callback_##name + +#define msgpack_unpack_object void* + +#define msgpack_unpack_user msgpack_unpack_t + + +struct template_context; + +static void template_func_init(struct template_context* ctx); + +static void* template_func_data(struct template_context* ctx); + +static int template_func_execute(struct template_context* ctx, + const char* data, size_t len, size_t* off); + + +static inline void* template_callback_init(msgpack_unpack_t* x) +{ return NULL; } + +static inline void* template_callback_uint8(msgpack_unpack_t* x, uint8_t d) +{ return x->callback.unpack_uint8(x->data, d); } + +static inline void* template_callback_uint16(msgpack_unpack_t* x, uint16_t d) +{ return x->callback.unpack_uint16(x->data, d); } + +static inline void* template_callback_uint32(msgpack_unpack_t* x, uint32_t d) +{ return x->callback.unpack_uint32(x->data, d); } + +static inline void* template_callback_uint64(msgpack_unpack_t* x, uint64_t d) +{ return x->callback.unpack_uint64(x->data, d); } + +static inline void* template_callback_int8(msgpack_unpack_t* x, int8_t d) +{ return x->callback.unpack_int8(x->data, d); } + +static inline void* template_callback_int16(msgpack_unpack_t* x, int16_t d) +{ return x->callback.unpack_int16(x->data, d); } + +static inline void* template_callback_int32(msgpack_unpack_t* x, int32_t d) +{ return x->callback.unpack_int32(x->data, d); } + +static inline void* template_callback_int64(msgpack_unpack_t* x, int64_t d) +{ return x->callback.unpack_int64(x->data, d); } + +static inline void* template_callback_float(msgpack_unpack_t* x, float d) +{ return x->callback.unpack_float(x->data, d); } + +static inline void* template_callback_double(msgpack_unpack_t* x, double d) +{ return x->callback.unpack_double(x->data, d); } + +static inline void* template_callback_nil(msgpack_unpack_t* x) +{ return x->callback.unpack_nil(x->data); } + +static inline void* template_callback_true(msgpack_unpack_t* x) +{ return x->callback.unpack_true(x->data); } + +static inline void* template_callback_false(msgpack_unpack_t* x) +{ return x->callback.unpack_false(x->data); } + +static inline void* template_callback_array(msgpack_unpack_t* x, unsigned int n) +{ return x->callback.unpack_array(x->data, n); } + +static inline void template_callback_array_item(msgpack_unpack_t* x, void* c, void* o) +{ x->callback.unpack_array_item(x->data, c, o); } + +static inline void* template_callback_map(msgpack_unpack_t* x, unsigned int n) +{ return x->callback.unpack_map(x->data, n); } + +static inline void template_callback_map_item(msgpack_unpack_t* x, void* c, void* k, void* v) +{ x->callback.unpack_map_item(x->data, c, k, v); } + +static inline void* template_callback_raw(msgpack_unpack_t* x, const char* b, const char* p, unsigned int l) +{ return x->callback.unpack_raw(x->data, b, p, l); } + + +#include "msgpack/unpack_template.h" + + msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback) { - msgpack_unpacker* ctx; - ctx = (msgpack_unpacker*)calloc(1, sizeof(msgpack_unpacker)); + struct template_context* ctx; + ctx = (struct template_context*)calloc(1, sizeof(struct template_context)); if(ctx == NULL) { return NULL; } - msgpack_unpacker_init(ctx); + template_func_init(ctx); ((msgpack_unpack_t*)ctx)->data = data; ((msgpack_unpack_t*)ctx)->callback = *callback; return (msgpack_unpack_t*)ctx; @@ -32,25 +118,27 @@ msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callba void msgpack_unpack_free(msgpack_unpack_t* ctx) { - free((msgpack_unpacker*)ctx); + free((struct template_context*)ctx); } void* msgpack_unpack_data(msgpack_unpack_t* ctx) { - return msgpack_unpacker_data((msgpack_unpacker*)ctx); + return template_func_data((struct template_context*)ctx); } void msgpack_unpack_reset(msgpack_unpack_t* ctx) { - msgpack_unpack_t x = ((msgpack_unpacker*)ctx)->user; - msgpack_unpacker_init((msgpack_unpacker*)ctx); - ((msgpack_unpacker*)ctx)->user = x; + msgpack_unpack_t x = ((struct template_context*)ctx)->user; + template_func_init((struct template_context*)ctx); + ((struct template_context*)ctx)->user = x; } -int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off) +int msgpack_unpack_execute(msgpack_unpack_t* ctx, + const char* data, size_t len, size_t* off) { - return msgpack_unpacker_execute( - (msgpack_unpacker*)ctx, + return template_func_execute( + (struct template_context*)ctx, data, len, off); } + diff --git a/c/unpack.h b/c/unpack.h index 6367439..c1cacab 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -21,24 +21,29 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct { - void* (*unpack_unsigned_int_8)(void* data, uint8_t d); - void* (*unpack_unsigned_int_16)(void* data, uint16_t d); - void* (*unpack_unsigned_int_32)(void* data, uint32_t d); - void* (*unpack_unsigned_int_64)(void* data, uint64_t d); - void* (*unpack_signed_int_8)(void* data, int8_t d); - void* (*unpack_signed_int_16)(void* data, int16_t d); - void* (*unpack_signed_int_32)(void* data, int32_t d); - void* (*unpack_signed_int_64)(void* data, int64_t d); + void* (*unpack_uint8)(void* data, uint8_t d); + void* (*unpack_uint16)(void* data, uint16_t d); + void* (*unpack_uint32)(void* data, uint32_t d); + void* (*unpack_uint64)(void* data, uint64_t d); + void* (*unpack_int8)(void* data, int8_t d); + void* (*unpack_int16)(void* data, int16_t d); + void* (*unpack_int32)(void* data, int32_t d); + void* (*unpack_int64)(void* data, int64_t d); void* (*unpack_float)(void* data, float d); void* (*unpack_double)(void* data, double d); void* (*unpack_nil)(void* data); void* (*unpack_true)(void* data); void* (*unpack_false)(void* data); - void* (*unpack_array_start)(void* data, unsigned int n); + void* (*unpack_array)(void* data, unsigned int n); void (*unpack_array_item)(void* data, void* c, void* o); - void* (*unpack_map_start)(void* data, unsigned int n); - void (*unpack_map_item)(void* data, void* c, void* k, void* v); + void* (*unpack_map)(void* data, unsigned int n); + void (*unpack_map_item)(void* data, void* c, void* k, void* v); void* (*unpack_raw)(void* data, const char* b, const char* p, unsigned int l); } msgpack_unpack_callback; @@ -51,8 +56,14 @@ msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callba void msgpack_unpack_free(msgpack_unpack_t* ctx); void msgpack_unpack_reset(msgpack_unpack_t* ctx); -int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off); +int msgpack_unpack_execute(msgpack_unpack_t* ctx, + const char* data, size_t len, size_t* off); void* msgpack_unpack_data(msgpack_unpack_t* ctx); + +#ifdef __cplusplus +} +#endif + #endif /* msgpack/unpack.h */ diff --git a/c/unpack_context.h b/c/unpack_context.h deleted file mode 100644 index d7b0388..0000000 --- a/c/unpack_context.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * MessagePack unpacking routine for C - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef UNPACK_CONTEXT_H__ -#define UNPACK_CONTEXT_H__ - -#include "msgpack/unpack.h" - -typedef void* msgpack_object; - -typedef msgpack_unpack_t msgpack_unpack_context; - -#include "msgpack/unpack/inline_context.h" - -#endif /* unpack_context.h */ - diff --git a/c/unpack_inline.c b/c/unpack_inline.c deleted file mode 100644 index 5282282..0000000 --- a/c/unpack_inline.c +++ /dev/null @@ -1,79 +0,0 @@ -/* - * MessagePack unpacking routine for C - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "unpack_context.h" - -static inline void* msgpack_unpack_init(msgpack_unpack_t* x) -{ return NULL; } - -static inline void* msgpack_unpack_unsigned_int_8(msgpack_unpack_t* x, uint8_t d) -{ return x->callback.unpack_unsigned_int_8(x->data, d); } - -static inline void* msgpack_unpack_unsigned_int_16(msgpack_unpack_t* x, uint16_t d) -{ return x->callback.unpack_unsigned_int_16(x->data, d); } - -static inline void* msgpack_unpack_unsigned_int_32(msgpack_unpack_t* x, uint32_t d) -{ return x->callback.unpack_unsigned_int_32(x->data, d); } - -static inline void* msgpack_unpack_unsigned_int_64(msgpack_unpack_t* x, uint64_t d) -{ return x->callback.unpack_unsigned_int_64(x->data, d); } - -static inline void* msgpack_unpack_signed_int_8(msgpack_unpack_t* x, int8_t d) -{ return x->callback.unpack_signed_int_8(x->data, d); } - -static inline void* msgpack_unpack_signed_int_16(msgpack_unpack_t* x, int16_t d) -{ return x->callback.unpack_signed_int_16(x->data, d); } - -static inline void* msgpack_unpack_signed_int_32(msgpack_unpack_t* x, int32_t d) -{ return x->callback.unpack_signed_int_32(x->data, d); } - -static inline void* msgpack_unpack_signed_int_64(msgpack_unpack_t* x, int64_t d) -{ return x->callback.unpack_signed_int_64(x->data, d); } - -static inline void* msgpack_unpack_float(msgpack_unpack_t* x, float d) -{ return x->callback.unpack_float(x->data, d); } - -static inline void* msgpack_unpack_double(msgpack_unpack_t* x, double d) -{ return x->callback.unpack_double(x->data, d); } - -static inline void* msgpack_unpack_nil(msgpack_unpack_t* x) -{ return x->callback.unpack_nil(x->data); } - -static inline void* msgpack_unpack_true(msgpack_unpack_t* x) -{ return x->callback.unpack_true(x->data); } - -static inline void* msgpack_unpack_false(msgpack_unpack_t* x) -{ return x->callback.unpack_false(x->data); } - -static inline void* msgpack_unpack_array_start(msgpack_unpack_t* x, unsigned int n) -{ return x->callback.unpack_array_start(x->data, n); } - -static inline void msgpack_unpack_array_item(msgpack_unpack_t* x, void* c, void* o) -{ x->callback.unpack_array_item(x->data, c, o); } - -static inline void* msgpack_unpack_map_start(msgpack_unpack_t* x, unsigned int n) -{ return x->callback.unpack_map_start(x->data, n); } - -static inline void msgpack_unpack_map_item(msgpack_unpack_t* x, void* c, void* k, void* v) -{ x->callback.unpack_map_item(x->data, c, k, v); } - -static inline void* msgpack_unpack_raw(msgpack_unpack_t* x, const char* b, const char* p, unsigned int l) -{ return x->callback.unpack_raw(x->data, b, p, l); } - - -#include "msgpack/unpack/inline_impl.h" - diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 08ea21f..aa22cc7 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -3,7 +3,6 @@ lib_LTLIBRARIES = libmsgpack.la libmsgpack_la_SOURCES = \ object.cpp \ unpack.cpp \ - unpack_inline.cpp \ zone.cpp nobase_include_HEADERS = \ @@ -14,13 +13,11 @@ nobase_include_HEADERS = \ msgpack/zone.hpp noinst_HEADERS = \ - unpack_context.hpp \ msgpack/zone.hpp.erb # FIXME object.lo: msgpack/zone.hpp unpack.lo: msgpack/zone.hpp -unpack_context.lo: msgpack/zone.hpp zone.lo: msgpack/zone.hpp msgpack/zone.hpp: msgpack/zone.hpp.erb diff --git a/cpp/bench.cpp b/cpp/bench.cpp index 4133a2e..517a870 100644 --- a/cpp/bench.cpp +++ b/cpp/bench.cpp @@ -6,10 +6,10 @@ #include #include -static const unsigned int TASK_INT_NUM = 1<<24; -static const unsigned int TASK_STR_LEN = 1<<15; -//static const unsigned int TASK_INT_NUM = 1<<23; -//static const unsigned int TASK_STR_LEN = 1<<14; +//static const unsigned int TASK_INT_NUM = 1<<24; +//static const unsigned int TASK_STR_LEN = 1<<15; +static const unsigned int TASK_INT_NUM = 1<<22; +static const unsigned int TASK_STR_LEN = 1<<13; static const char* TASK_STR_PTR; @@ -24,7 +24,7 @@ public: + (double)(endtime.tv_usec - m_timeval.tv_usec) / 1000 / 1000; std::cout << sec << " sec" << std::endl; std::cout << (double(bufsz)/1024/1024) << " MB" << std::endl; - std::cout << (bufsz/sec/1024/1024*8) << " Mbps" << std::endl; + std::cout << (bufsz/sec/1000/1000*8) << " Mbps" << std::endl; } private: timeval m_timeval; diff --git a/cpp/bench.mk b/cpp/bench.mk new file mode 100644 index 0000000..52a4b80 --- /dev/null +++ b/cpp/bench.mk @@ -0,0 +1,9 @@ + +CXXFLAGS += -Wall -g -I. -I.. -O4 +LDFLAGS += + +all: bench + +bench: bench.o unpack.o zone.o object.o pack.hpp unpack.hpp zone.hpp object.hpp + $(CXX) bench.o unpack.o zone.o object.o $(CFLAGS) $(LDFLAGS) -o $@ + diff --git a/cpp/object.cpp b/cpp/object.cpp index 7bb9841..ef2a68c 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -155,35 +155,35 @@ inline void numeric_pack(dynamic_packer& p, V v); template <> inline void numeric_pack(dynamic_packer& p, uint8_t v) - { p.pack_unsigned_int_8(v); } + { p.pack_uint8(v); } template <> inline void numeric_pack(dynamic_packer& p, uint16_t v) - { p.pack_unsigned_int_16(v); } + { p.pack_uint16(v); } template <> inline void numeric_pack(dynamic_packer& p, uint32_t v) - { p.pack_unsigned_int_32(v); } + { p.pack_uint32(v); } template <> inline void numeric_pack(dynamic_packer& p, uint64_t v) - { p.pack_unsigned_int_64(v); } + { p.pack_uint64(v); } template <> inline void numeric_pack(dynamic_packer& p, int8_t v) - { p.pack_unsigned_int_8(v); } + { p.pack_int8(v); } template <> inline void numeric_pack(dynamic_packer& p, int16_t v) - { p.pack_unsigned_int_16(v); } + { p.pack_int16(v); } template <> inline void numeric_pack(dynamic_packer& p, int32_t v) - { p.pack_unsigned_int_32(v); } + { p.pack_int32(v); } template <> inline void numeric_pack(dynamic_packer& p, int64_t v) - { p.pack_unsigned_int_64(v); } + { p.pack_int64(v); } template <> inline void numeric_pack(dynamic_packer& p, float v) diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 9580679..f3eeb34 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -34,14 +34,14 @@ public: public: void pack_int(int d) { pack_int_impl(m_stream, d); } void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } - void pack_unsigned_int_8(uint8_t d) { pack_unsigned_int_8_impl(m_stream, d); } - void pack_unsigned_int_16(uint16_t d) { pack_unsigned_int_16_impl(m_stream, d); } - void pack_unsigned_int_32(uint32_t d) { pack_unsigned_int_32_impl(m_stream, d); } - void pack_unsigned_int_64(uint64_t d) { pack_unsigned_int_64_impl(m_stream, d); } - void pack_signed_int_8(uint8_t d) { pack_signed_int_8_impl(m_stream, d); } - void pack_signed_int_16(uint16_t d) { pack_signed_int_16_impl(m_stream, d); } - void pack_signed_int_32(uint32_t d) { pack_signed_int_32_impl(m_stream, d); } - void pack_signed_int_64(uint64_t d) { pack_signed_int_64_impl(m_stream, d); } + void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } + void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } + void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } + void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } + void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } + void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } + void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } + void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } void pack_float(float d) { pack_float_impl(m_stream, d); } void pack_double(double d) { pack_double_impl(m_stream, d); } void pack_nil() { pack_nil(m_stream); } @@ -49,20 +49,19 @@ public: void pack_false() { pack_false(m_stream); } void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } - void pack_string(const char* b) { pack_string_impl(m_stream, b); } void pack_raw(const char* b, size_t l) { pack_raw_impl(m_stream, (const void*)b, l); } private: static void pack_int_impl(Stream& x, int d); static void pack_unsigned_int_impl(Stream& x, unsigned int d); - static void pack_unsigned_int_8_impl(Stream& x, uint8_t d); - static void pack_unsigned_int_16_impl(Stream& x, uint16_t d); - static void pack_unsigned_int_32_impl(Stream& x, uint32_t d); - static void pack_unsigned_int_64_impl(Stream& x, uint64_t d); - static void pack_signed_int_8_impl(Stream& x, int8_t d); - static void pack_signed_int_16_impl(Stream& x, int16_t d); - static void pack_signed_int_32_impl(Stream& x, int32_t d); - static void pack_signed_int_64_impl(Stream& x, int64_t d); + static void pack_uint8_impl(Stream& x, uint8_t d); + static void pack_uint16_impl(Stream& x, uint16_t d); + static void pack_uint32_impl(Stream& x, uint32_t d); + static void pack_uint64_impl(Stream& x, uint64_t d); + static void pack_int8_impl(Stream& x, int8_t d); + static void pack_int16_impl(Stream& x, int16_t d); + static void pack_int32_impl(Stream& x, int32_t d); + static void pack_int64_impl(Stream& x, int64_t d); static void pack_float_impl(Stream& x, float d); static void pack_double_impl(Stream& x, double d); static void pack_nil_impl(Stream& x); @@ -70,7 +69,6 @@ private: static void pack_false_impl(Stream& x); static void pack_array_impl(Stream& x, unsigned int n); static void pack_map_impl(Stream& x, unsigned int n); - static void pack_string_impl(Stream& x, const char* b); static void pack_raw_impl(Stream& x, const void* b, size_t l); static void append_buffer(Stream& x, const unsigned char* buf, unsigned int len) { x.write((const char*)buf, len); } @@ -85,11 +83,10 @@ private: #define msgpack_pack_inline_func(name) \ template \ inline void packer::pack_ ## name ## _impl -#define msgpack_pack_context Stream& +#define msgpack_pack_user Stream& #define msgpack_pack_append_buffer append_buffer -#include "msgpack/pack/inline_impl.h" -#undef msgpack_pack_context -#undef msgpack_pack_append_buffer +#include "msgpack/pack_template.h" + template packer::packer(Stream& s) : m_stream(s) { } @@ -118,14 +115,14 @@ public: public: void pack_int(int d) { pack_int_impl(m_stream, d); } void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } - void pack_unsigned_int_8(uint8_t d) { pack_unsigned_int_8_impl(m_stream, d); } - void pack_unsigned_int_16(uint16_t d) { pack_unsigned_int_16_impl(m_stream, d); } - void pack_unsigned_int_32(uint32_t d) { pack_unsigned_int_32_impl(m_stream, d); } - void pack_unsigned_int_64(uint64_t d) { pack_unsigned_int_64_impl(m_stream, d); } - void pack_signed_int_8(uint8_t d) { pack_signed_int_8_impl(m_stream, d); } - void pack_signed_int_16(uint16_t d) { pack_signed_int_16_impl(m_stream, d); } - void pack_signed_int_32(uint32_t d) { pack_signed_int_32_impl(m_stream, d); } - void pack_signed_int_64(uint64_t d) { pack_signed_int_64_impl(m_stream, d); } + void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } + void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } + void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } + void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } + void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } + void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } + void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } + void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } void pack_float(float d) { pack_float_impl(m_stream, d); } void pack_double(double d) { pack_double_impl(m_stream, d); } void pack_nil() { pack_nil_impl(m_stream); } @@ -139,14 +136,14 @@ public: private: static void pack_int_impl(dynamic_stream& x, int d); static void pack_unsigned_int_impl(dynamic_stream& x, unsigned int d); - static void pack_unsigned_int_8_impl(dynamic_stream& x, uint8_t d); - static void pack_unsigned_int_16_impl(dynamic_stream& x, uint16_t d); - static void pack_unsigned_int_32_impl(dynamic_stream& x, uint32_t d); - static void pack_unsigned_int_64_impl(dynamic_stream& x, uint64_t d); - static void pack_signed_int_8_impl(dynamic_stream& x, int8_t d); - static void pack_signed_int_16_impl(dynamic_stream& x, int16_t d); - static void pack_signed_int_32_impl(dynamic_stream& x, int32_t d); - static void pack_signed_int_64_impl(dynamic_stream& x, int64_t d); + static void pack_uint8_impl(dynamic_stream& x, uint8_t d); + static void pack_uint16_impl(dynamic_stream& x, uint16_t d); + static void pack_uint32_impl(dynamic_stream& x, uint32_t d); + static void pack_uint64_impl(dynamic_stream& x, uint64_t d); + static void pack_int8_impl(dynamic_stream& x, int8_t d); + static void pack_int16_impl(dynamic_stream& x, int16_t d); + static void pack_int32_impl(dynamic_stream& x, int32_t d); + static void pack_int64_impl(dynamic_stream& x, int64_t d); static void pack_float_impl(dynamic_stream& x, float d); static void pack_double_impl(dynamic_stream& x, double d); static void pack_nil_impl(dynamic_stream& x); @@ -166,14 +163,11 @@ private: dynamic_packer(); }; -#undef MSGPACK_PACK_INLINE_IMPL_H__ #define msgpack_pack_inline_func(name) \ inline void dynamic_packer::pack_ ## name ## _impl -#define msgpack_pack_context dynamic_stream& +#define msgpack_pack_user dynamic_stream& #define msgpack_pack_append_buffer append_buffer -#include "msgpack/pack/inline_impl.h" -#undef msgpack_pack_context -#undef msgpack_pack_append_buffer +#include "msgpack/pack_template.h" template dynamic_packer::dynamic_packer(Stream& s) : m_stream(s) { } diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index fe7f4b5..873d3da 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -16,12 +16,97 @@ // limitations under the License. // #include "msgpack/unpack.hpp" -#include "unpack_context.hpp" +#include "msgpack/unpack_define.h" #include namespace msgpack { +#define msgpack_unpack_struct(name) \ + struct msgpack_unpacker_##name + +#define msgpack_unpack_func(ret, name) \ + ret msgpack_unpacker_##name + +#define msgpack_unpack_callback(name) \ + msgpack_unpack_##name + +#define msgpack_unpack_object object_class* + +#define msgpack_unpack_user zone* + + +struct msgpack_unpacker_context; + +static void msgpack_unpacker_init(struct msgpack_unpacker_context* ctx); + +static object_class* msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); + +static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, + const char* data, size_t len, size_t* off); + + +static inline object_class* msgpack_unpack_init(zone** z) +{ return NULL; } + +static inline object_class* msgpack_unpack_uint8(zone** z, uint8_t d) +{ return (*z)->nu8(d); } + +static inline object_class* msgpack_unpack_uint16(zone** z, uint16_t d) +{ return (*z)->nu16(d); } + +static inline object_class* msgpack_unpack_uint32(zone** z, uint32_t d) +{ return (*z)->nu32(d); } + +static inline object_class* msgpack_unpack_uint64(zone** z, uint64_t d) +{ return (*z)->nu64(d); } + +static inline object_class* msgpack_unpack_int8(zone** z, int8_t d) +{ return (*z)->ni8(d); } + +static inline object_class* msgpack_unpack_int16(zone** z, int16_t d) +{ return (*z)->ni16(d); } + +static inline object_class* msgpack_unpack_int32(zone** z, int32_t d) +{ return (*z)->ni32(d); } + +static inline object_class* msgpack_unpack_int64(zone** z, int64_t d) +{ return (*z)->ni64(d); } + +static inline object_class* msgpack_unpack_float(zone** z, float d) +{ return (*z)->nfloat(d); } + +static inline object_class* msgpack_unpack_double(zone** z, double d) +{ return (*z)->ndouble(d); } + +static inline object_class* msgpack_unpack_nil(zone** z) +{ return (*z)->nnil(); } + +static inline object_class* msgpack_unpack_true(zone** z) +{ return (*z)->ntrue(); } + +static inline object_class* msgpack_unpack_false(zone** z) +{ return (*z)->nfalse(); } + +static inline object_class* msgpack_unpack_array(zone** z, unsigned int n) +{ return (*z)->narray(n); } + +static inline void msgpack_unpack_array_item(zone** z, object_class* c, object_class* o) +{ reinterpret_cast(c)->push_back(o); } + +static inline object_class* msgpack_unpack_map(zone** z, unsigned int n) +{ return (*z)->nmap(); } + +static inline void msgpack_unpack_map_item(zone** z, object_class* c, object_class* k, object_class* v) +{ reinterpret_cast(c)->store(k, v); } + +static inline object_class* msgpack_unpack_raw(zone** z, const char* b, const char* p, unsigned int l) +{ return (*z)->nraw_ref(p, l); } + + +#include "msgpack/unpack_template.h" + + struct unpacker::context { context(zone* z) { @@ -65,7 +150,7 @@ struct unpacker::context { } private: - msgpack_unpacker m_ctx; + msgpack_unpacker_context m_ctx; private: context(); @@ -171,17 +256,26 @@ void unpacker::reset() } -object unpacker::unpack(const char* data, size_t len, zone& z) +object unpacker::unpack(const char* data, size_t len, zone& z, size_t* off) { context ctx(&z); - size_t off = 0; - int ret = ctx.execute(data, len, &off); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret == 0) { - throw unpack_error("insufficient bytes"); - } else if(off < len) { - throw unpack_error("extra bytes"); + if(off) { + int ret = ctx.execute(data, len, off); + if(ret < 0) { + throw unpack_error("parse error"); + } else if(ret == 0) { + throw unpack_error("insufficient bytes"); + } + } else { + size_t noff = 0; + int ret = ctx.execute(data, len, &noff); + if(ret < 0) { + throw unpack_error("parse error"); + } else if(ret == 0) { + throw unpack_error("insufficient bytes"); + } else if(noff < len) { + throw unpack_error("extra bytes"); + } } return ctx.data(); } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 473e8e9..86cfb6e 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -100,7 +100,7 @@ private: unpacker(const unpacker&); public: - static object unpack(const char* data, size_t len, zone& z); + static object unpack(const char* data, size_t len, zone& z, size_t* off = NULL); }; @@ -136,9 +136,9 @@ inline void unpacker::remove_nonparsed_buffer() { m_used = m_off; } -inline object unpack(const char* data, size_t len, zone& z) +inline object unpack(const char* data, size_t len, zone& z, size_t* off = NULL) { - return unpacker::unpack(data, len, z); + return unpacker::unpack(data, len, z, off); } diff --git a/cpp/unpack_context.hpp b/cpp/unpack_context.hpp deleted file mode 100644 index 59bd872..0000000 --- a/cpp/unpack_context.hpp +++ /dev/null @@ -1,31 +0,0 @@ -// -// MessagePack for C++ deserializing routine -// -// Copyright (C) 2008 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#ifndef UNPACK_CONTEXT_HPP__ -#define UNPACK_CONTEXT_HPP__ - -#include "msgpack/zone.hpp" -#include "msgpack/object.hpp" - -typedef msgpack::object_class* msgpack_object; - -typedef msgpack::zone* msgpack_unpack_context; - -#include "msgpack/unpack/inline_context.h" - -#endif /* unpack_context.h */ - diff --git a/cpp/unpack_inline.cpp b/cpp/unpack_inline.cpp deleted file mode 100644 index 40f2769..0000000 --- a/cpp/unpack_inline.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// -// MessagePack for C++ deserializing routine -// -// Copyright (C) 2008 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#include "unpack_context.hpp" - - -extern "C" { -using namespace msgpack; - - -static inline object_class* msgpack_unpack_init(zone** z) -{ return NULL; } - -static inline object_class* msgpack_unpack_unsigned_int_8(zone** z, uint8_t d) -{ return (*z)->nu8(d); } - -static inline object_class* msgpack_unpack_unsigned_int_16(zone** z, uint16_t d) -{ return (*z)->nu16(d); } - -static inline object_class* msgpack_unpack_unsigned_int_32(zone** z, uint32_t d) -{ return (*z)->nu32(d); } - -static inline object_class* msgpack_unpack_unsigned_int_64(zone** z, uint64_t d) -{ return (*z)->nu64(d); } - -static inline object_class* msgpack_unpack_signed_int_8(zone** z, int8_t d) -{ return (*z)->ni8(d); } - -static inline object_class* msgpack_unpack_signed_int_16(zone** z, int16_t d) -{ return (*z)->ni16(d); } - -static inline object_class* msgpack_unpack_signed_int_32(zone** z, int32_t d) -{ return (*z)->ni32(d); } - -static inline object_class* msgpack_unpack_signed_int_64(zone** z, int64_t d) -{ return (*z)->ni64(d); } - -static inline object_class* msgpack_unpack_float(zone** z, float d) -{ return (*z)->nfloat(d); } - -static inline object_class* msgpack_unpack_double(zone** z, double d) -{ return (*z)->ndouble(d); } - -static inline object_class* msgpack_unpack_nil(zone** z) -{ return (*z)->nnil(); } - -static inline object_class* msgpack_unpack_true(zone** z) -{ return (*z)->ntrue(); } - -static inline object_class* msgpack_unpack_false(zone** z) -{ return (*z)->nfalse(); } - -static inline object_class* msgpack_unpack_array_start(zone** z, unsigned int n) -{ return (*z)->narray(n); } - -static inline void msgpack_unpack_array_item(zone** z, object_class* c, object_class* o) -{ reinterpret_cast(c)->push_back(o); } - -static inline object_class* msgpack_unpack_map_start(zone** z, unsigned int n) -{ return (*z)->nmap(); } - -static inline void msgpack_unpack_map_item(zone** z, object_class* c, object_class* k, object_class* v) -{ reinterpret_cast(c)->store(k, v); } - -static inline object_class* msgpack_unpack_raw(zone** z, const char* b, const char* p, unsigned int l) -{ return (*z)->nraw_ref(p, l); } - - -} // extern "C" - -#include "msgpack/unpack/inline_impl.h" - diff --git a/msgpack/pack/inline_impl.h b/msgpack/pack_template.h similarity index 77% rename from msgpack/pack/inline_impl.h rename to msgpack/pack_template.h index 635b697..22d9911 100644 --- a/msgpack/pack/inline_impl.h +++ b/msgpack/pack_template.h @@ -1,5 +1,5 @@ /* - * MessagePack packing routine + * MessagePack packing routine template * * Copyright (C) 2008 FURUHASHI Sadayuki * @@ -15,8 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MSGPACK_PACK_INLINE_IMPL_H__ -#define MSGPACK_PACK_INLINE_IMPL_H__ #if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) #if __BYTE_ORDER == __LITTLE_ENDIAN @@ -52,17 +50,26 @@ #endif + #ifndef msgpack_pack_inline_func -#define msgpack_pack_inline_func(name) \ - inline void msgpack_pack_##name +#error msgpack_pack_inline_func template is not defined #endif +#ifndef msgpack_pack_user +#error msgpack_pack_user type is not defined +#endif + +#ifndef msgpack_pack_append_buffer +#error msgpack_pack_append_buffer callback is not defined +#endif + + /* * Integer */ // wrapper -msgpack_pack_inline_func(int)(msgpack_pack_context x, int d) +msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) { if(d < -32) { if(d < -32768) { // signed 32 @@ -95,7 +102,7 @@ msgpack_pack_inline_func(int)(msgpack_pack_context x, int d) } // wrapper -msgpack_pack_inline_func(unsigned_int)(msgpack_pack_context x, unsigned int d) +msgpack_pack_inline_func(unsigned_int)(msgpack_pack_user x, unsigned int d) { if(d < 128) { // fixnum @@ -115,7 +122,7 @@ msgpack_pack_inline_func(unsigned_int)(msgpack_pack_context x, unsigned int d) } } -msgpack_pack_inline_func(unsigned_int_8)(msgpack_pack_context x, uint8_t d) +msgpack_pack_inline_func(uint8)(msgpack_pack_user x, uint8_t d) { if(d < 128) { msgpack_pack_append_buffer(x, &d, 1); @@ -125,26 +132,26 @@ msgpack_pack_inline_func(unsigned_int_8)(msgpack_pack_context x, uint8_t d) } } -msgpack_pack_inline_func(unsigned_int_16)(msgpack_pack_context x, uint16_t d) +msgpack_pack_inline_func(uint16)(msgpack_pack_user x, uint16_t d) { const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func(unsigned_int_32)(msgpack_pack_context x, uint32_t d) +msgpack_pack_inline_func(uint32)(msgpack_pack_user x, uint32_t d) { const unsigned char buf[5] = {0xce, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func(unsigned_int_64)(msgpack_pack_context x, uint64_t d) +msgpack_pack_inline_func(uint64)(msgpack_pack_user x, uint64_t d) { // FIXME optimization const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } -msgpack_pack_inline_func(signed_int_8)(msgpack_pack_context x, int8_t d) +msgpack_pack_inline_func(int8)(msgpack_pack_user x, int8_t d) { if(d > 0) { msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); @@ -156,19 +163,19 @@ msgpack_pack_inline_func(signed_int_8)(msgpack_pack_context x, int8_t d) } } -msgpack_pack_inline_func(signed_int_16)(msgpack_pack_context x, int16_t d) +msgpack_pack_inline_func(int16)(msgpack_pack_user x, int16_t d) { const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func(signed_int_32)(msgpack_pack_context x, int32_t d) +msgpack_pack_inline_func(int32)(msgpack_pack_user x, int32_t d) { const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func(signed_int_64)(msgpack_pack_context x, int64_t d) +msgpack_pack_inline_func(int64)(msgpack_pack_user x, int64_t d) { // FIXME optimization const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; @@ -180,14 +187,14 @@ msgpack_pack_inline_func(signed_int_64)(msgpack_pack_context x, int64_t d) * Float */ -msgpack_pack_inline_func(float)(msgpack_pack_context x, float d) +msgpack_pack_inline_func(float)(msgpack_pack_user x, float d) { uint32_t n = *((uint32_t*)&d); // FIXME const unsigned char buf[5] = {0xca, STORE_BE32(n)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func(double)(msgpack_pack_context x, double d) +msgpack_pack_inline_func(double)(msgpack_pack_user x, double d) { uint64_t n = *((uint64_t*)&d); // FIXME const unsigned char buf[9] = {0xcb, STORE_BE64(n)}; @@ -199,7 +206,7 @@ msgpack_pack_inline_func(double)(msgpack_pack_context x, double d) * Nil */ -msgpack_pack_inline_func(nil)(msgpack_pack_context x) +msgpack_pack_inline_func(nil)(msgpack_pack_user x) { static const unsigned char d = 0xc0; msgpack_pack_append_buffer(x, &d, 1); @@ -210,13 +217,13 @@ msgpack_pack_inline_func(nil)(msgpack_pack_context x) * Boolean */ -msgpack_pack_inline_func(true)(msgpack_pack_context x) +msgpack_pack_inline_func(true)(msgpack_pack_user x) { static const unsigned char d = 0xc3; msgpack_pack_append_buffer(x, &d, 1); } -msgpack_pack_inline_func(false)(msgpack_pack_context x) +msgpack_pack_inline_func(false)(msgpack_pack_user x) { static const unsigned char d = 0xc2; msgpack_pack_append_buffer(x, &d, 1); @@ -227,7 +234,7 @@ msgpack_pack_inline_func(false)(msgpack_pack_context x) * Array */ -msgpack_pack_inline_func(array)(msgpack_pack_context x, unsigned int n) +msgpack_pack_inline_func(array)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x90 | n; @@ -248,7 +255,7 @@ msgpack_pack_inline_func(array)(msgpack_pack_context x, unsigned int n) * Map */ -msgpack_pack_inline_func(map)(msgpack_pack_context x, unsigned int n) +msgpack_pack_inline_func(map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; @@ -269,13 +276,7 @@ msgpack_pack_inline_func(map)(msgpack_pack_context x, unsigned int n) * Raw */ -msgpack_pack_inline_func(string)(msgpack_pack_context x, const char* b) -{ - uint32_t l = strlen(b); - msgpack_pack_append_buffer(x, (const unsigned char*)b, l+1); -} - -msgpack_pack_inline_func(raw)(msgpack_pack_context x, const void* b, size_t l) +msgpack_pack_inline_func(raw)(msgpack_pack_user x, const void* b, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; @@ -294,10 +295,10 @@ msgpack_pack_inline_func(raw)(msgpack_pack_context x, const void* b, size_t l) #undef msgpack_pack_inline_func +#undef msgpack_pack_user +#undef msgpack_pack_append_buffer #undef STORE_BE16 #undef STORE_BE32 #undef STORE_BE64 -#endif /* msgpack/pack/inline_impl.h */ - diff --git a/msgpack/unpack/callback.h b/msgpack/unpack/callback.h deleted file mode 100644 index 315bb9e..0000000 --- a/msgpack/unpack/callback.h +++ /dev/null @@ -1,24 +0,0 @@ - -msgpack_object msgpack_unpack_init(msgpack_unpack_context* x); -msgpack_object msgpack_unpack_unsigned_int_8(msgpack_unpack_context* x, uint8_t d); -msgpack_object msgpack_unpack_unsigned_int_16(msgpack_unpack_context* x, uint16_t d); -msgpack_object msgpack_unpack_unsigned_int_32(msgpack_unpack_context* x, uint32_t d); -msgpack_object msgpack_unpack_unsigned_int_64(msgpack_unpack_context* x, uint64_t d); -msgpack_object msgpack_unpack_signed_int_8(msgpack_unpack_context* x, int8_t d); -msgpack_object msgpack_unpack_signed_int_16(msgpack_unpack_context* x, int16_t d); -msgpack_object msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int32_t d); -msgpack_object msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d); -msgpack_object msgpack_unpack_float(msgpack_unpack_context* x, float d); -msgpack_object msgpack_unpack_double(msgpack_unpack_context* x, double d); -msgpack_object msgpack_unpack_big_int(msgpack_unpack_context* x, const void* b, unsigned int l); -msgpack_object msgpack_unpack_big_float(msgpack_unpack_context* x, const void* b, unsigned int l); -msgpack_object msgpack_unpack_nil(msgpack_unpack_context* x); -msgpack_object msgpack_unpack_true(msgpack_unpack_context* x); -msgpack_object msgpack_unpack_false(msgpack_unpack_context* x); -msgpack_object msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned int n); - void msgpack_unpack_array_item(msgpack_unpack_context* x, msgpack_object c, msgpack_object o); -msgpack_object msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n); - void msgpack_unpack_map_item(msgpack_unpack_context* x, msgpack_object c, msgpack_object k, msgpack_object v); -msgpack_object msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l); -msgpack_object msgpack_unpack_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l); - diff --git a/msgpack/unpack/inline_context.h b/msgpack/unpack/inline_context.h deleted file mode 100644 index e764c09..0000000 --- a/msgpack/unpack/inline_context.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * MessagePack unpacking routine - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MSGPACK_UNPACK_INLINE_CONTEXT_H__ -#define MSGPACK_UNPACK_INLINE_CONTEXT_H__ - -#include -#include - -#ifndef MSG_STACK_SIZE -#define MSG_STACK_SIZE 16 -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - msgpack_object obj; - size_t count; - unsigned int ct; - union { - /*const unsigned char* terminal_trail_start;*/ - msgpack_object map_key; - } tmp; -} msgpack_unpacker_stack; - -typedef struct { - msgpack_unpack_context user; // must be first - unsigned int cs; - unsigned int trail; - unsigned int top; - msgpack_unpacker_stack stack[MSG_STACK_SIZE]; -} msgpack_unpacker; - -void msgpack_unpacker_init(msgpack_unpacker* ctx); -int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len, size_t* off); -#define msgpack_unpacker_data(unpacker) (unpacker)->stack[0].obj - -#ifdef __cplusplus -} -#endif - -#endif /* msgpack/unpack/inline_context.h */ - diff --git a/msgpack/unpack/inline_impl.h b/msgpack/unpack/inline_impl.h deleted file mode 100644 index a6557f9..0000000 --- a/msgpack/unpack/inline_impl.h +++ /dev/null @@ -1,465 +0,0 @@ -/* - * MessagePack unpacking routine - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MSGPACK_UNPACK_INLINE_IMPL_H__ -#define MSGPACK_UNPACK_INLINE_IMPL_H__ - -#include -#include -#include -/*#include */ - -#ifdef __cplusplus -extern "C" { -#endif - -// Positive FixNum 0xxxxxxx 0x00 - 0x7f -// Negative FixNum 111xxxxx 0xe0 - 0xff -// Variable 110xxxxx 0xc0 - 0xdf -// nil 00000 0xc0 -// string 00001 0xc1 -// false 00010 0xc2 -// true 00011 0xc3 -// (?) 00100 0xc4 -// (?) 00101 0xc5 -// (?) 00110 0xc6 -// (?) 00111 0xc7 -// (?) 01000 0xc8 -// (?) 01001 0xc9 -// float 01010 0xca -// double 01011 0xcb -// uint 8 01100 0xcc -// uint 16 01101 0xcd -// uint 32 01110 0xce -// uint 64 01111 0xcf -// int 8 10000 0xd0 -// int 16 10001 0xd1 -// int 32 10010 0xd2 -// int 64 10011 0xd3 -// (?) 10100 0xd4 -// (?) 10101 0xd5 -// (big float 16) 10110 0xd6 -// (big float 32) 10111 0xd7 -// (big integer 16) 11000 0xd8 -// (big integer 32) 11001 0xd9 -// raw 16 11010 0xda -// raw 32 11011 0xdb -// array 16 11100 0xdc -// array 32 11101 0xdd -// map 16 11110 0xde -// map 32 11111 0xdf -// FixRaw 101xxxxx 0xa0 - 0xbf -// FixArray 1001xxxx 0x90 - 0x9f -// FixMap 1000xxxx 0x80 - 0x8f - - -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - -#define betoh16(x) ntohs(x) -#define betoh32(x) ntohl(x) - -#ifdef __LITTLE_ENDIAN__ -#if defined(__bswap_64) -# define betoh64(x) __bswap_64(x) -#elif defined(__DARWIN_OSSwapInt64) -# define betoh64(x) __DARWIN_OSSwapInt64(x) -#else -static inline uint64_t betoh64(uint64_t x) { - return ((x << 56) & 0xff00000000000000ULL ) | - ((x << 40) & 0x00ff000000000000ULL ) | - ((x << 24) & 0x0000ff0000000000ULL ) | - ((x << 8) & 0x000000ff00000000ULL ) | - ((x >> 8) & 0x00000000ff000000ULL ) | - ((x >> 24) & 0x0000000000ff0000ULL ) | - ((x >> 40) & 0x000000000000ff00ULL ) | - ((x >> 56) & 0x00000000000000ffULL ) ; -} -#endif -#else -#define betoh64(x) (x) -#endif - - -typedef enum { - CS_HEADER = 0x00, // nil - - //CS_STRING = 0x01, - //CS_ = 0x02, // false - //CS_ = 0x03, // true - - //CS_ = 0x04, - //CS_ = 0x05, - //CS_ = 0x06, - //CS_ = 0x07, - - //CS_ = 0x08, - //CS_ = 0x09, - CS_FLOAT = 0x0a, - CS_DOUBLE = 0x0b, - CS_UNSIGNED_INT_8 = 0x0c, - CS_UNSIGNED_INT_16 = 0x0d, - CS_UNSIGNED_INT_32 = 0x0e, - CS_UNSIGNED_INT_64 = 0x0f, - CS_SIGNED_INT_8 = 0x10, - CS_SIGNED_INT_16 = 0x11, - CS_SIGNED_INT_32 = 0x12, - CS_SIGNED_INT_64 = 0x13, - - //CS_ = 0x14, - //CS_ = 0x15, - //CS_BIG_INT_16 = 0x16, - //CS_BIG_INT_32 = 0x17, - //CS_BIG_FLOAT_16 = 0x18, - //CS_BIG_FLOAT_32 = 0x19, - CS_RAW_16 = 0x1a, - CS_RAW_32 = 0x1b, - CS_ARRAY_16 = 0x1c, - CS_ARRAY_32 = 0x1d, - CS_MAP_16 = 0x1e, - CS_MAP_32 = 0x1f, - - //ACS_BIG_INT_VALUE, - //ACS_BIG_FLOAT_VALUE, - ACS_RAW_VALUE, -} current_state_t; - - -typedef enum { - CT_ARRAY_ITEM, - CT_MAP_KEY, - CT_MAP_VALUE, -} container_type_t; - - -void msgpack_unpacker_init(msgpack_unpacker* ctx) -{ - memset(ctx, 0, sizeof(msgpack_unpacker)); // FIXME init ctx->user? - ctx->cs = CS_HEADER; - ctx->trail = 0; - ctx->top = 0; - ctx->stack[0].obj = msgpack_unpack_init(&ctx->user); -} - -int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len, size_t* off) -{ - assert(len >= *off); - - const unsigned char* p = (unsigned char*)data + *off; - const unsigned char* const pe = (unsigned char*)data + len; - const void* n = NULL; - - unsigned int trail = ctx->trail; - unsigned int cs = ctx->cs; - unsigned int top = ctx->top; - msgpack_unpacker_stack* stack = ctx->stack; - msgpack_unpack_context* user = &ctx->user; - - msgpack_object obj; - msgpack_unpacker_stack* c = NULL; - - int ret; - -#define push_simple_value(func) \ - obj = func(user); \ - /*printf("obj %d\n",obj);*/ \ - goto _push -#define push_fixed_value(func, arg) \ - obj = func(user, arg); \ - /*printf("obj %d\n",obj);*/ \ - goto _push -#define push_variable_value(func, base, pos, len) \ - obj = func(user, (const char*)base, (const char*)pos, len); \ - /*printf("obj %d\n",obj);*/ \ - goto _push - -/* -#define again_terminal_trail(_cs, from) \ - cs = _cs; \ - stack[top].tmp.terminal_trail_start = from; \ - goto _terminal_trail_again -*/ -#define again_fixed_trail(_cs, trail_len) \ - trail = trail_len; \ - cs = _cs; \ - goto _fixed_trail_again -#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ - trail = trail_len; \ - if(trail == 0) { goto ifzero; } \ - cs = _cs; \ - goto _fixed_trail_again - -#define start_container(func, count_, ct_) \ - stack[top].obj = func(user, count_); \ - if((count_) == 0) { obj = stack[top].obj; goto _push; } \ - if(top >= MSG_STACK_SIZE) { goto _failed; } \ - stack[top].ct = ct_; \ - stack[top].count = count_; \ - /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ - /*printf("stack push %d\n", top);*/ \ - ++top; \ - goto _header_again - -#define NEXT_CS(p) \ - ((unsigned int)*p & 0x1f) - -#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) betoh16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) betoh32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) betoh64(*(uint64_t*)ptr) - - if(p == pe) { goto _out; } - do { - switch(cs) { - case CS_HEADER: - switch(*p) { - case 0x00 ... 0x7f: // Positive Fixnum - push_fixed_value(msgpack_unpack_unsigned_int_8, *(uint8_t*)p); - case 0xe0 ... 0xff: // Negative Fixnum - push_fixed_value(msgpack_unpack_signed_int_8, *(int8_t*)p); - case 0xc0 ... 0xdf: // Variable - switch(*p) { - case 0xc0: // nil - push_simple_value(msgpack_unpack_nil); - //case 0xc1: // string - // again_terminal_trail(NEXT_CS(p), p+1); - case 0xc2: // false - push_simple_value(msgpack_unpack_false); - case 0xc3: // true - push_simple_value(msgpack_unpack_true); - //case 0xc4: - //case 0xc5: - //case 0xc6: - //case 0xc7: - //case 0xc8: - //case 0xc9: - case 0xca: // float - case 0xcb: // double - case 0xcc: // unsigned int 8 - case 0xcd: // unsigned int 16 - case 0xce: // unsigned int 32 - case 0xcf: // unsigned int 64 - case 0xd0: // signed int 8 - case 0xd1: // signed int 16 - case 0xd2: // signed int 32 - case 0xd3: // signed int 64 - again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); - //case 0xd4: - //case 0xd5: - //case 0xd6: // big integer 16 - //case 0xd7: // big integer 32 - //case 0xd8: // big float 16 - //case 0xd9: // big float 32 - case 0xda: // raw 16 - case 0xdb: // raw 32 - case 0xdc: // array 16 - case 0xdd: // array 32 - case 0xde: // map 16 - case 0xdf: // map 32 - again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); - default: - goto _failed; - } - case 0xa0 ... 0xbf: // FixRaw - again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); - case 0x90 ... 0x9f: // FixArray - start_container(msgpack_unpack_array_start, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); - case 0x80 ... 0x8f: // FixMap - start_container(msgpack_unpack_map_start, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); - - default: - goto _failed; - } - // end CS_HEADER - - - //_terminal_trail_again: - // ++p; - - //case CS_STRING: - // if(*p == 0) { - // const unsigned char* start = stack[top].tmp.terminal_trail_start; - // obj = msgpack_unpack_string(user, start, p-start); - // goto _push; - // } - // goto _terminal_trail_again; - - - _fixed_trail_again: - ++p; - - default: - if((size_t)(pe - p) < trail) { goto _out; } - n = p; p += trail - 1; - switch(cs) { - //case CS_ - //case CS_ - case CS_FLOAT: { - uint32_t x = PTR_CAST_32(n); // FIXME - push_fixed_value(msgpack_unpack_float, *((float*)&x)); } - case CS_DOUBLE: { - uint64_t x = PTR_CAST_64(n); // FIXME - push_fixed_value(msgpack_unpack_double, *((double*)&x)); } - case CS_UNSIGNED_INT_8: - push_fixed_value(msgpack_unpack_unsigned_int_8, (uint8_t)PTR_CAST_8(n)); - case CS_UNSIGNED_INT_16: - push_fixed_value(msgpack_unpack_unsigned_int_16, (uint16_t)PTR_CAST_16(n)); - case CS_UNSIGNED_INT_32: - push_fixed_value(msgpack_unpack_unsigned_int_32, (uint32_t)PTR_CAST_32(n)); - case CS_UNSIGNED_INT_64: - push_fixed_value(msgpack_unpack_unsigned_int_64, (uint64_t)PTR_CAST_64(n)); - - case CS_SIGNED_INT_8: - push_fixed_value(msgpack_unpack_signed_int_8, (int8_t)PTR_CAST_8(n)); - case CS_SIGNED_INT_16: - push_fixed_value(msgpack_unpack_signed_int_16, (int16_t)PTR_CAST_16(n)); - case CS_SIGNED_INT_32: - push_fixed_value(msgpack_unpack_signed_int_32, (int32_t)PTR_CAST_32(n)); - case CS_SIGNED_INT_64: - push_fixed_value(msgpack_unpack_signed_int_64, (int64_t)PTR_CAST_64(n)); - - //case CS_ - //case CS_ - //case CS_BIG_INT_16: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); - //case CS_BIG_INT_32: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); - //case ACS_BIG_INT_VALUE: - //_big_int_zero: - // // FIXME - // push_variable_value(msgpack_unpack_big_int, data, n, trail); - - //case CS_BIG_FLOAT_16: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); - //case CS_BIG_FLOAT_32: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); - //case ACS_BIG_FLOAT_VALUE: - //_big_float_zero: - // // FIXME - // push_variable_value(msgpack_unpack_big_float, data, n, trail); - - case CS_RAW_16: - again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); - case CS_RAW_32: - again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); - case ACS_RAW_VALUE: - _raw_zero: - push_variable_value(msgpack_unpack_raw, data, n, trail); - - case CS_ARRAY_16: - start_container(msgpack_unpack_array_start, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); - case CS_ARRAY_32: - start_container(msgpack_unpack_array_start, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); - - case CS_MAP_16: - start_container(msgpack_unpack_map_start, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); - case CS_MAP_32: - start_container(msgpack_unpack_map_start, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); - - default: - goto _failed; - } - } - -_push: - if(top == 0) { goto _finish; } - c = &stack[top-1]; - switch(c->ct) { - case CT_ARRAY_ITEM: - msgpack_unpack_array_item(user, c->obj, obj); - if(--c->count == 0) { - obj = c->obj; - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - goto _header_again; - case CT_MAP_KEY: - c->tmp.map_key = obj; - c->ct = CT_MAP_VALUE; - goto _header_again; - case CT_MAP_VALUE: - msgpack_unpack_map_item(user, c->obj, c->tmp.map_key, obj); - if(--c->count == 0) { - obj = c->obj; - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - c->ct = CT_MAP_KEY; - goto _header_again; - - default: - goto _failed; - } - -_header_again: - cs = CS_HEADER; - ++p; - } while(p != pe); - goto _out; - - -_finish: - stack[0].obj = obj; - ++p; - ret = 1; - /*printf("-- finish --\n"); */ - goto _end; - -_failed: - /*printf("** FAILED **\n"); */ - ret = -1; - goto _end; - -_out: - ret = 0; - goto _end; - -_end: - ctx->cs = cs; - ctx->trail = trail; - ctx->top = top; - *off = p - (const unsigned char*)data; - - return ret; -} - - -#ifdef betoh16 -#undef betoh16 -#endif - -#ifdef betoh32 -#undef betoh32 -#endif - -#ifdef betoh64 -#undef betoh64 -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* msgpack/unpack/inline_impl.h */ - diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h new file mode 100644 index 0000000..1d9db19 --- /dev/null +++ b/msgpack/unpack_define.h @@ -0,0 +1,127 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_UNPACK_DEFINE_H__ +#define MSGPACK_UNPACK_DEFINE_H__ + +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifndef MSGPACK_MAX_STACK_SIZE +#define MSGPACK_MAX_STACK_SIZE 16 +#endif + + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#define msgpack_betoh16(x) ntohs(x) +#define msgpack_betoh32(x) ntohl(x) + +#ifdef __LITTLE_ENDIAN__ +#if defined(__bswap_64) +# define msgpack_betoh64(x) __bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) +#else +static inline uint64_t msgpack_betoh64(uint64_t x) { + return ((x << 56) & 0xff00000000000000ULL ) | + ((x << 40) & 0x00ff000000000000ULL ) | + ((x << 24) & 0x0000ff0000000000ULL ) | + ((x << 8) & 0x000000ff00000000ULL ) | + ((x >> 8) & 0x00000000ff000000ULL ) | + ((x >> 24) & 0x0000000000ff0000ULL ) | + ((x >> 40) & 0x000000000000ff00ULL ) | + ((x >> 56) & 0x00000000000000ffULL ) ; +} +#endif +#else +#define msgpack_betoh64(x) (x) +#endif + + +typedef enum { + CS_HEADER = 0x00, // nil + + //CS_ = 0x01, + //CS_ = 0x02, // false + //CS_ = 0x03, // true + + //CS_ = 0x04, + //CS_ = 0x05, + //CS_ = 0x06, + //CS_ = 0x07, + + //CS_ = 0x08, + //CS_ = 0x09, + CS_FLOAT = 0x0a, + CS_DOUBLE = 0x0b, + CS_UINT_8 = 0x0c, + CS_UINT_16 = 0x0d, + CS_UINT_32 = 0x0e, + CS_UINT_64 = 0x0f, + CS_INT_8 = 0x10, + CS_INT_16 = 0x11, + CS_INT_32 = 0x12, + CS_INT_64 = 0x13, + + //CS_ = 0x14, + //CS_ = 0x15, + //CS_BIG_INT_16 = 0x16, + //CS_BIG_INT_32 = 0x17, + //CS_BIG_FLOAT_16 = 0x18, + //CS_BIG_FLOAT_32 = 0x19, + CS_RAW_16 = 0x1a, + CS_RAW_32 = 0x1b, + CS_ARRAY_16 = 0x1c, + CS_ARRAY_32 = 0x1d, + CS_MAP_16 = 0x1e, + CS_MAP_32 = 0x1f, + + //ACS_BIG_INT_VALUE, + //ACS_BIG_FLOAT_VALUE, + ACS_RAW_VALUE, +} msgpack_unpack_state; + + +typedef enum { + CT_ARRAY_ITEM, + CT_MAP_KEY, + CT_MAP_VALUE, +} msgpack_container_type; + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/unpack_define.h */ + diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h new file mode 100644 index 0000000..8197fa4 --- /dev/null +++ b/msgpack/unpack_template.h @@ -0,0 +1,360 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef msgpack_unpack_func +#error msgpack_unpack_func template is not defined +#endif + +#ifndef msgpack_unpack_callback +#error msgpack_unpack_callback template is not defined +#endif + +#ifndef msgpack_unpack_struct +#error msgpack_unpack_struct template is not defined +#endif + +#ifndef msgpack_unpack_struct_decl +#define msgpack_unpack_struct_decl(name) msgpack_unpack_struct(name) +#endif + +#ifndef msgpack_unpack_object +#error msgpack_unpack_object type is not defined +#endif + +#ifndef msgpack_unpack_user +#error msgpack_unpack_user type is not defined +#endif + + +msgpack_unpack_struct_decl(stack) { + msgpack_unpack_object obj; + size_t count; + unsigned int ct; + msgpack_unpack_object map_key; +}; + +msgpack_unpack_struct_decl(context) { + msgpack_unpack_user user; // must be first + unsigned int cs; + unsigned int trail; + unsigned int top; + msgpack_unpack_struct(stack) stack[MSGPACK_MAX_STACK_SIZE]; +}; + + +msgpack_unpack_func(void, init)(msgpack_unpack_struct(context)* ctx) +{ + /*memset(ctx, 0, sizeof( msgpack_unpack_struct(context) )); FIXME needed? */ + ctx->cs = CS_HEADER; + ctx->trail = 0; + ctx->top = 0; + ctx->stack[0].obj = msgpack_unpack_callback(init)(&ctx->user); +} + +msgpack_unpack_func(msgpack_unpack_object, data)(msgpack_unpack_struct(context)* unpacker) +{ + return (unpacker)->stack[0].obj; +} + + +msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const char* data, size_t len, size_t* off) +{ + assert(len >= *off); + + const unsigned char* p = (unsigned char*)data + *off; + const unsigned char* const pe = (unsigned char*)data + len; + const void* n = NULL; + + unsigned int trail = ctx->trail; + unsigned int cs = ctx->cs; + unsigned int top = ctx->top; + msgpack_unpack_struct(stack)* stack = ctx->stack; + msgpack_unpack_user* user = &ctx->user; + + msgpack_unpack_object obj; + msgpack_unpack_struct(stack)* c = NULL; + + int ret; + +#define push_simple_value(func) \ + obj = msgpack_unpack_callback(func)(user); \ + /*printf("obj %d\n",obj);*/ \ + goto _push +#define push_fixed_value(func, arg) \ + obj = msgpack_unpack_callback(func)(user, arg); \ + /*printf("obj %d\n",obj);*/ \ + goto _push +#define push_variable_value(func, base, pos, len) \ + obj = msgpack_unpack_callback(func)(user, (const char*)base, (const char*)pos, len); \ + /*printf("obj %d\n",obj);*/ \ + goto _push + +#define again_fixed_trail(_cs, trail_len) \ + trail = trail_len; \ + cs = _cs; \ + goto _fixed_trail_again +#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ + trail = trail_len; \ + if(trail == 0) { goto ifzero; } \ + cs = _cs; \ + goto _fixed_trail_again + +#define start_container(func, count_, ct_) \ + stack[top].obj = msgpack_unpack_callback(func)(user, count_); \ + if((count_) == 0) { obj = stack[top].obj; goto _push; } \ + if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ + stack[top].ct = ct_; \ + stack[top].count = count_; \ + /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ + /*printf("stack push %d\n", top);*/ \ + ++top; \ + goto _header_again + +#define NEXT_CS(p) \ + ((unsigned int)*p & 0x1f) + +#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) +#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) + + if(p == pe) { goto _out; } + do { + switch(cs) { + case CS_HEADER: + switch(*p) { + case 0x00 ... 0x7f: // Positive Fixnum + push_fixed_value(uint8, *(uint8_t*)p); + case 0xe0 ... 0xff: // Negative Fixnum + push_fixed_value(int8, *(int8_t*)p); + case 0xc0 ... 0xdf: // Variable + switch(*p) { + case 0xc0: // nil + push_simple_value(nil); + //case 0xc1: // string + // again_terminal_trail(NEXT_CS(p), p+1); + case 0xc2: // false + push_simple_value(false); + case 0xc3: // true + push_simple_value(true); + //case 0xc4: + //case 0xc5: + //case 0xc6: + //case 0xc7: + //case 0xc8: + //case 0xc9: + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); + //case 0xd4: + //case 0xd5: + //case 0xd6: // big integer 16 + //case 0xd7: // big integer 32 + //case 0xd8: // big float 16 + //case 0xd9: // big float 32 + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); + default: + goto _failed; + } + case 0xa0 ... 0xbf: // FixRaw + again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); + case 0x90 ... 0x9f: // FixArray + start_container(array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); + case 0x80 ... 0x8f: // FixMap + start_container(map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); + + default: + goto _failed; + } + // end CS_HEADER + + + _fixed_trail_again: + ++p; + + default: + if((size_t)(pe - p) < trail) { goto _out; } + n = p; p += trail - 1; + switch(cs) { + //case CS_ + //case CS_ + case CS_FLOAT: { + uint32_t x = PTR_CAST_32(n); // FIXME + push_fixed_value(float, *((float*)&x)); } + case CS_DOUBLE: { + uint64_t x = PTR_CAST_64(n); // FIXME + push_fixed_value(double, *((double*)&x)); } + case CS_UINT_8: + push_fixed_value(uint8, (uint8_t)PTR_CAST_8(n)); + case CS_UINT_16: + push_fixed_value(uint16, (uint16_t)PTR_CAST_16(n)); + case CS_UINT_32: + push_fixed_value(uint32, (uint32_t)PTR_CAST_32(n)); + case CS_UINT_64: + push_fixed_value(uint64, (uint64_t)PTR_CAST_64(n)); + + case CS_INT_8: + push_fixed_value(int8, (int8_t)PTR_CAST_8(n)); + case CS_INT_16: + push_fixed_value(int16, (int16_t)PTR_CAST_16(n)); + case CS_INT_32: + push_fixed_value(int32, (int32_t)PTR_CAST_32(n)); + case CS_INT_64: + push_fixed_value(int64, (int64_t)PTR_CAST_64(n)); + + //case CS_ + //case CS_ + //case CS_BIG_INT_16: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); + //case CS_BIG_INT_32: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); + //case ACS_BIG_INT_VALUE: + //_big_int_zero: + // // FIXME + // push_variable_value(big_int, data, n, trail); + + //case CS_BIG_FLOAT_16: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); + //case CS_BIG_FLOAT_32: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); + //case ACS_BIG_FLOAT_VALUE: + //_big_float_zero: + // // FIXME + // push_variable_value(big_float, data, n, trail); + + case CS_RAW_16: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); + case CS_RAW_32: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); + case ACS_RAW_VALUE: + _raw_zero: + push_variable_value(raw, data, n, trail); + + case CS_ARRAY_16: + start_container(array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + case CS_ARRAY_32: + start_container(array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + + case CS_MAP_16: + start_container(map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + case CS_MAP_32: + start_container(map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + + default: + goto _failed; + } + } + +_push: + if(top == 0) { goto _finish; } + c = &stack[top-1]; + switch(c->ct) { + case CT_ARRAY_ITEM: + msgpack_unpack_callback(array_item)(user, c->obj, obj); + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + goto _header_again; + case CT_MAP_KEY: + c->map_key = obj; + c->ct = CT_MAP_VALUE; + goto _header_again; + case CT_MAP_VALUE: + msgpack_unpack_callback(map_item)(user, c->obj, c->map_key, obj); + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + c->ct = CT_MAP_KEY; + goto _header_again; + + default: + goto _failed; + } + +_header_again: + cs = CS_HEADER; + ++p; + } while(p != pe); + goto _out; + + +_finish: + stack[0].obj = obj; + ++p; + ret = 1; + /*printf("-- finish --\n"); */ + goto _end; + +_failed: + /*printf("** FAILED **\n"); */ + ret = -1; + goto _end; + +_out: + ret = 0; + goto _end; + +_end: + ctx->cs = cs; + ctx->trail = trail; + ctx->top = top; + *off = p - (const unsigned char*)data; + + return ret; +} + + +#undef msgpack_unpack_func +#undef msgpack_unpack_callback +#undef msgpack_unpack_struct +#undef msgpack_unpack_object +#undef msgpack_unpack_user + +#undef push_simple_value +#undef push_fixed_value +#undef push_variable_value +#undef again_fixed_trail +#undef again_fixed_trail_if_zero +#undef start_container + +#undef NEXT_CS +#undef PTR_CAST_8 +#undef PTR_CAST_16 +#undef PTR_CAST_32 +#undef PTR_CAST_64 + diff --git a/ruby/gem/Manifest.txt b/ruby/gem/Manifest.txt index 299a4e0..388135e 100644 --- a/ruby/gem/Manifest.txt +++ b/ruby/gem/Manifest.txt @@ -7,15 +7,12 @@ config/requirements.rb ext/extconf.rb ext/pack.c ext/pack.h -ext/pack_inline.h ext/rbinit.c ext/unpack.c ext/unpack.h -ext/unpack_context.h -ext/unpack_inline.c -msgpack/pack/inline_impl.h -msgpack/unpack/inline_context.h -msgpack/unpack/inline_impl.h +msgpack/pack_template.h +msgpack/unpack_define.h +msgpack/unpack_template.h lib/msgpack/version.rb script/console script/destroy diff --git a/ruby/gem/lib/msgpack/version.rb b/ruby/gem/lib/msgpack/version.rb index c65972f..b2a5db6 100644 --- a/ruby/gem/lib/msgpack/version.rb +++ b/ruby/gem/lib/msgpack/version.rb @@ -1,7 +1,7 @@ module MessagePack module VERSION #:nodoc: MAJOR = 0 - MINOR = 1 + MINOR = 2 TINY = 0 STRING = [MAJOR, MINOR, TINY].join('.') diff --git a/ruby/gengem.sh b/ruby/gengem.sh index 4f8623c..c8abcc2 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -3,16 +3,13 @@ cp extconf.rb gem/ext/ cp pack.c gem/ext/ cp pack.h gem/ext/ -cp pack_inline.h gem/ext/ cp rbinit.c gem/ext/ cp unpack.c gem/ext/ cp unpack.h gem/ext/ -cp unpack_context.h gem/ext/ -cp unpack_inline.c gem/ext/ cp ../README gem/README.txt -cp ../msgpack/pack/inline_impl.h gem/msgpack/pack/ -cp ../msgpack/unpack/inline_context.h gem/msgpack/unpack/ -cp ../msgpack/unpack/inline_impl.h gem/msgpack/unpack/ +cp ../msgpack/pack_template.h gem/msgpack/ +cp ../msgpack/unpack_define.h gem/msgpack/ +cp ../msgpack/unpack_template.h gem/msgpack/ cd gem && rake --trace package diff --git a/ruby/pack.c b/ruby/pack.c index 3d71776..54f610c 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -15,7 +15,41 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "pack_inline.h" +#include "ruby.h" +#include +#include + +#define msgpack_pack_inline_func(name) \ + static void msgpack_pack_##name + +#define msgpack_pack_user VALUE + +#define msgpack_pack_append_buffer(user, buf, len) \ + rb_str_buf_cat(user, (const void*)buf, len) + +/* +static void msgpack_pack_int(VALUE x, int d); +static void msgpack_pack_unsigned_int(VALUE x, unsigned int d); +static void msgpack_pack_uint8(VALUE x, uint8_t d); +static void msgpack_pack_uint16(VALUE x, uint16_t d); +static void msgpack_pack_uint32(VALUE x, uint32_t d); +static void msgpack_pack_uint64(VALUE x, uint64_t d); +static void msgpack_pack_int8(VALUE x, int8_t d); +static void msgpack_pack_int16(VALUE x, int16_t d); +static void msgpack_pack_int32(VALUE x, int32_t d); +static void msgpack_pack_int64(VALUE x, int64_t d); +static void msgpack_pack_float(VALUE x, float d); +static void msgpack_pack_double(VALUE x, double d); +static void msgpack_pack_nil(VALUE x); +static void msgpack_pack_true(VALUE x); +static void msgpack_pack_false(VALUE x); +static void msgpack_pack_array(VALUE x, unsigned int n); +static void msgpack_pack_map(VALUE x, unsigned int n); +static void msgpack_pack_raw(VALUE x, const void* b, size_t l); +*/ + +#include "msgpack/pack_template.h" + #ifndef RUBY_VM #include "st.h" // ruby hash @@ -72,9 +106,9 @@ static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self) ARG_BUFFER(out, argc, argv); // FIXME bignum if(RBIGNUM_SIGN(self)) { // positive - msgpack_pack_unsigned_int_64(out, rb_big2ull(self)); + msgpack_pack_uint64(out, rb_big2ull(self)); } else { // negative - msgpack_pack_signed_int_64(out, rb_big2ll(self)); + msgpack_pack_int64(out, rb_big2ll(self)); } return out; } diff --git a/ruby/pack_inline.h b/ruby/pack_inline.h deleted file mode 100644 index bda74c3..0000000 --- a/ruby/pack_inline.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * MessagePack packing routine for Ruby - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef PACK_INLINE_H__ -#define PACK_INLINE_H__ - -#include "ruby.h" - -typedef VALUE msgpack_pack_context; - -static inline void msgpack_pack_append_buffer(VALUE x, const unsigned char* b, unsigned int l) -{ - rb_str_buf_cat(x, (const void*)b, l); -} - -#include -#include /* __BYTE_ORDER */ -#include "msgpack/pack/inline_impl.h" - -#endif /* pack_inline.h */ - diff --git a/ruby/unpack.c b/ruby/unpack.c index fa2996d..8439c02 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -16,8 +16,99 @@ * limitations under the License. */ #include "ruby.h" -#include "unpack_context.h" -#include +#include "msgpack/unpack_define.h" + + +typedef struct { + int finished; +} msgpack_unpack_context; + + +#define msgpack_unpack_struct(name) \ + struct msgpack_unpacker_##name + +#define msgpack_unpack_func(ret, name) \ + ret msgpack_unpacker_##name + +#define msgpack_unpack_callback(name) \ + template_callback_##name + +#define msgpack_unpack_object VALUE + +#define msgpack_unpack_user msgpack_unpack_context + + +struct msgpack_unpacker_context; +typedef struct msgpack_unpacker_context msgpack_unpacker; + +static void msgpack_unpacker_init(msgpack_unpacker* ctx); + +static VALUE msgpack_unpacker_data(msgpack_unpacker* ctx); + +static int msgpack_unpacker_execute(msgpack_unpacker* ctx, + const char* data, size_t len, size_t* off); + + +static inline VALUE template_callback_init(msgpack_unpack_context* x) +{ return Qnil; } + +static inline VALUE template_callback_uint8(msgpack_unpack_context* x, uint8_t d) +{ return INT2FIX(d); } + +static inline VALUE template_callback_uint16(msgpack_unpack_context* x, uint16_t d) +{ return INT2FIX(d); } + +static inline VALUE template_callback_uint32(msgpack_unpack_context* x, uint32_t d) +{ return UINT2NUM(d); } + +static inline VALUE template_callback_uint64(msgpack_unpack_context* x, uint64_t d) +{ return rb_ull2inum(d); } + +static inline VALUE template_callback_int8(msgpack_unpack_context* x, int8_t d) +{ return INT2FIX((long)d); } + +static inline VALUE template_callback_int16(msgpack_unpack_context* x, int16_t d) +{ return INT2FIX((long)d); } + +static inline VALUE template_callback_int32(msgpack_unpack_context* x, int32_t d) +{ return INT2NUM((long)d); } + +static inline VALUE template_callback_int64(msgpack_unpack_context* x, int64_t d) +{ return rb_ll2inum(d); } + +static inline VALUE template_callback_float(msgpack_unpack_context* x, float d) +{ return rb_float_new(d); } + +static inline VALUE template_callback_double(msgpack_unpack_context* x, double d) +{ return rb_float_new(d); } + +static inline VALUE template_callback_nil(msgpack_unpack_context* x) +{ return Qnil; } + +static inline VALUE template_callback_true(msgpack_unpack_context* x) +{ return Qtrue; } + +static inline VALUE template_callback_false(msgpack_unpack_context* x) +{ return Qfalse; } + +static inline VALUE template_callback_array(msgpack_unpack_context* x, unsigned int n) +{ return rb_ary_new2(n); } + +static inline void template_callback_array_item(msgpack_unpack_context* x, VALUE c, VALUE o) +{ rb_ary_push(c, o); } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] + +static inline VALUE template_callback_map(msgpack_unpack_context* x, unsigned int n) +{ return rb_hash_new(); } + +static inline void template_callback_map_item(msgpack_unpack_context* x, VALUE c, VALUE k, VALUE v) +{ rb_hash_aset(c, k, v); } + +static inline VALUE template_callback_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l) +{ return rb_str_new(p, l); } + + +#include "msgpack/unpack_template.h" + #define UNPACKER(from, name) \ msgpack_unpacker *name = NULL; \ @@ -45,7 +136,7 @@ static void MessagePack_Unpacker_mark(msgpack_unpacker *mp) unsigned int i; for(i=0; i < mp->top; ++i) { rb_gc_mark(mp->stack[i].obj); - rb_gc_mark(mp->stack[i].tmp.map_key); + rb_gc_mark(mp->stack[i].map_key); } } @@ -61,7 +152,7 @@ static VALUE MessagePack_Unpacker_alloc(VALUE klass) static VALUE MessagePack_Unpacker_reset(VALUE self) { UNPACKER(self, mp); - mp->user.finished = false; + mp->user.finished = 0; msgpack_unpacker_init(mp); return self; } @@ -93,10 +184,10 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE args) if(ret < 0) { rb_raise(eUnpackError, "Parse error."); } else if(ret > 0) { - mp->user.finished = true; + mp->user.finished = 1; return ULONG2NUM(from); } else { - mp->user.finished = false; + mp->user.finished = 0; return ULONG2NUM(from); } } diff --git a/ruby/unpack_context.h b/ruby/unpack_context.h deleted file mode 100644 index 35e0132..0000000 --- a/ruby/unpack_context.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * MessagePack unpacking routine for Ruby - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef UNPACK_CONTEXT_H__ -#define UNPACK_CONTEXT_H__ - -#include "ruby.h" -#include -#include - -typedef VALUE msgpack_object; - -typedef struct { - bool finished; -} msgpack_unpack_context; - - -#include "msgpack/unpack/inline_context.h" - -#endif /* unpack_context.h */ - diff --git a/ruby/unpack_inline.c b/ruby/unpack_inline.c deleted file mode 100644 index 849441c..0000000 --- a/ruby/unpack_inline.c +++ /dev/null @@ -1,78 +0,0 @@ -/* - * MessagePack unpacking routine for Ruby - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "unpack_context.h" - -static inline VALUE msgpack_unpack_init(msgpack_unpack_context* x) -{ return Qnil; } - -static inline VALUE msgpack_unpack_unsigned_int_8(msgpack_unpack_context* x, uint8_t d) -{ return INT2FIX(d); } - -static inline VALUE msgpack_unpack_unsigned_int_16(msgpack_unpack_context* x, uint16_t d) -{ return INT2FIX(d); } - -static inline VALUE msgpack_unpack_unsigned_int_32(msgpack_unpack_context* x, uint32_t d) -{ return UINT2NUM(d); } - -static inline VALUE msgpack_unpack_unsigned_int_64(msgpack_unpack_context* x, uint64_t d) -{ return rb_ull2inum(d); } - -static inline VALUE msgpack_unpack_signed_int_8(msgpack_unpack_context* x, int8_t d) -{ return INT2FIX((long)d); } - -static inline VALUE msgpack_unpack_signed_int_16(msgpack_unpack_context* x, int16_t d) -{ return INT2FIX((long)d); } - -static inline VALUE msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int32_t d) -{ return INT2NUM((long)d); } - -static inline VALUE msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d) -{ return rb_ll2inum(d); } - -static inline VALUE msgpack_unpack_float(msgpack_unpack_context* x, float d) -{ return rb_float_new(d); } - -static inline VALUE msgpack_unpack_double(msgpack_unpack_context* x, double d) -{ return rb_float_new(d); } - -static inline VALUE msgpack_unpack_nil(msgpack_unpack_context* x) -{ return Qnil; } - -static inline VALUE msgpack_unpack_true(msgpack_unpack_context* x) -{ return Qtrue; } - -static inline VALUE msgpack_unpack_false(msgpack_unpack_context* x) -{ return Qfalse; } - -static inline VALUE msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned int n) -{ return rb_ary_new2(n); } - -static inline void msgpack_unpack_array_item(msgpack_unpack_context* x, VALUE c, VALUE o) -{ rb_ary_push(c, o); } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] - -static inline VALUE msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n) -{ return rb_hash_new(); } - -static inline void msgpack_unpack_map_item(msgpack_unpack_context* x, VALUE c, VALUE k, VALUE v) -{ rb_hash_aset(c, k, v); } - -static inline VALUE msgpack_unpack_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l) -{ return rb_str_new(p, l); } - -#include "msgpack/unpack/inline_impl.h" - From 9923cf4daf631432e389dd0694042b6b405c1288 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0020/1172] lang/c/msgpack: reimplemented C++ binding with template-based static resolution design git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@67 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/unpack.c | 8 +- configure.in | 2 +- cpp/Makefile.am | 21 +- cpp/bench.cpp | 15 +- cpp/bench.mk | 2 +- cpp/object.cpp | 497 ++++++++------------------------------ cpp/object.hpp | 354 ++++++--------------------- cpp/pack.hpp | 10 +- cpp/test.cpp | 156 ++++++------ cpp/type.hpp | 9 + cpp/type/array.hpp | 59 +++++ cpp/type/boolean.hpp | 49 ++++ cpp/type/float.hpp | 67 +++++ cpp/type/integer.hpp | 243 +++++++++++++++++++ cpp/type/map.hpp | 132 ++++++++++ cpp/type/nil.hpp | 47 ++++ cpp/type/raw.hpp | 99 ++++++++ cpp/type/tuple.hpp.erb | 172 +++++++++++++ cpp/unpack.cpp | 149 ++++++------ cpp/unpack.hpp | 16 +- cpp/zone.cpp | 61 ++--- cpp/zone.hpp | 70 ++++++ cpp/zone.hpp.erb | 202 ---------------- msgpack/pack_template.h | 2 - msgpack/unpack_template.h | 4 +- ruby/unpack.c | 8 +- 26 files changed, 1338 insertions(+), 1116 deletions(-) create mode 100644 cpp/type.hpp create mode 100644 cpp/type/array.hpp create mode 100644 cpp/type/boolean.hpp create mode 100644 cpp/type/float.hpp create mode 100644 cpp/type/integer.hpp create mode 100644 cpp/type/map.hpp create mode 100644 cpp/type/nil.hpp create mode 100644 cpp/type/raw.hpp create mode 100644 cpp/type/tuple.hpp.erb create mode 100644 cpp/zone.hpp delete mode 100644 cpp/zone.hpp.erb diff --git a/c/unpack.c b/c/unpack.c index 3058427..c7f25c4 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -89,14 +89,14 @@ static inline void* template_callback_false(msgpack_unpack_t* x) static inline void* template_callback_array(msgpack_unpack_t* x, unsigned int n) { return x->callback.unpack_array(x->data, n); } -static inline void template_callback_array_item(msgpack_unpack_t* x, void* c, void* o) -{ x->callback.unpack_array_item(x->data, c, o); } +static inline void template_callback_array_item(msgpack_unpack_t* x, void** c, void* o) +{ x->callback.unpack_array_item(x->data, *c, o); } static inline void* template_callback_map(msgpack_unpack_t* x, unsigned int n) { return x->callback.unpack_map(x->data, n); } -static inline void template_callback_map_item(msgpack_unpack_t* x, void* c, void* k, void* v) -{ x->callback.unpack_map_item(x->data, c, k, v); } +static inline void template_callback_map_item(msgpack_unpack_t* x, void** c, void* k, void* v) +{ x->callback.unpack_map_item(x->data, *c, k, v); } static inline void* template_callback_raw(msgpack_unpack_t* x, const char* b, const char* p, unsigned int l) { return x->callback.unpack_raw(x->data, b, p, l); } diff --git a/configure.in b/configure.in index 9b6f6d4..33acfa6 100644 --- a/configure.in +++ b/configure.in @@ -1,4 +1,4 @@ -AC_INIT(msgpack/unpack/inline_impl.h) +AC_INIT(msgpack/unpack_template.h) AM_INIT_AUTOMAKE(msgpack, 0.1.0) AC_CONFIG_HEADER(config.h) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index aa22cc7..8b13bea 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -10,20 +10,27 @@ nobase_include_HEADERS = \ msgpack/pack.hpp \ msgpack/unpack.hpp \ msgpack/object.hpp \ - msgpack/zone.hpp + msgpack/zone.hpp \ + msgpack/type.hpp \ + msgpack/type/array.hpp \ + msgpack/type/boolean.hpp \ + msgpack/type/integer.hpp \ + msgpack/type/map.hpp \ + msgpack/type/nil.hpp \ + msgpack/type/tuple.hpp noinst_HEADERS = \ - msgpack/zone.hpp.erb + msgpack/type/tuple.hpp.erb # FIXME -object.lo: msgpack/zone.hpp -unpack.lo: msgpack/zone.hpp -zone.lo: msgpack/zone.hpp +object.lo: msgpack/type/tuple.hpp +unpack.lo: msgpack/type/tuple.hpp +zone.lo: msgpack/type/tuple.hpp -msgpack/zone.hpp: msgpack/zone.hpp.erb +msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb erb $< > $@ -MOSTLYCLEANFILES = zone.hpp +MOSTLYCLEANFILES = msgpack/type/tuple.hpp libmsgpack_la_LDFLAGS = -version-info 0:0:0 diff --git a/cpp/bench.cpp b/cpp/bench.cpp index 517a870..1aad807 100644 --- a/cpp/bench.cpp +++ b/cpp/bench.cpp @@ -6,10 +6,10 @@ #include #include -//static const unsigned int TASK_INT_NUM = 1<<24; -//static const unsigned int TASK_STR_LEN = 1<<15; -static const unsigned int TASK_INT_NUM = 1<<22; -static const unsigned int TASK_STR_LEN = 1<<13; +static const unsigned int TASK_INT_NUM = 1<<24; +static const unsigned int TASK_STR_LEN = 1<<15; +//static const unsigned int TASK_INT_NUM = 1<<22; +//static const unsigned int TASK_STR_LEN = 1<<13; static const char* TASK_STR_PTR; @@ -104,6 +104,7 @@ void bench_msgpack_int() } timer.show_stat(buf.size()); + std::cout << "----" << std::endl; std::cout << "unpack integer" << std::endl; @@ -116,6 +117,7 @@ void bench_msgpack_int() } timer.show_stat(buf.size()); + /* std::cout << "----" << std::endl; std::cout << "dynamic pack integer" << std::endl; @@ -124,6 +126,7 @@ void bench_msgpack_int() timer.reset(); msgpack::pack(buf, obj); timer.show_stat(buf.size()); + */ } void bench_msgpack_str() @@ -144,6 +147,7 @@ void bench_msgpack_str() } timer.show_stat(buf.size()); + std::cout << "----" << std::endl; std::cout << "unpack string" << std::endl; @@ -156,6 +160,8 @@ void bench_msgpack_str() } timer.show_stat(buf.size()); + + /* std::cout << "----" << std::endl; std::cout << "dynamic pack string" << std::endl; @@ -164,6 +170,7 @@ void bench_msgpack_str() timer.reset(); msgpack::pack(buf, obj); timer.show_stat(buf.size()); + */ } int main(void) diff --git a/cpp/bench.mk b/cpp/bench.mk index 52a4b80..8da2b7f 100644 --- a/cpp/bench.mk +++ b/cpp/bench.mk @@ -5,5 +5,5 @@ LDFLAGS += all: bench bench: bench.o unpack.o zone.o object.o pack.hpp unpack.hpp zone.hpp object.hpp - $(CXX) bench.o unpack.o zone.o object.o $(CFLAGS) $(LDFLAGS) -o $@ + $(CXX) bench.o unpack.o zone.o object.o $(CXXFLAGS) $(LDFLAGS) -o $@ diff --git a/cpp/object.cpp b/cpp/object.cpp index ef2a68c..276732c 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -16,417 +16,128 @@ // limitations under the License. // #include "msgpack/object.hpp" -#include "msgpack/pack.hpp" + +#include namespace msgpack { -namespace { -template -struct numeric_overflow_signed_impl; +std::ostream& operator<< (std::ostream& s, const object o) +{ + switch(o.type) { + case type::NIL: + s << "nil"; + break; -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) < sizeof(X) ) { - if( static_cast( std::numeric_limits::max()) < x ) { return 1; } - if( static_cast(-std::numeric_limits::max()) > x ) { return -1; } + case type::BOOLEAN: + s << (o.via.boolean ? "true" : "false"); + break; + + case type::POSITIVE_INTEGER: + s << o.via.u64; + break; + + case type::NEGATIVE_INTEGER: + s << o.via.i64; + break; + + case type::RAW: + (s << '"').write(o.via.ref.ptr, o.via.ref.size) << '"'; + break; + + case type::ARRAY: + s << "["; + if(o.via.container.size != 0) { + object* p(o.via.container.ptr); + s << *p; + ++p; + for(object* const pend(o.via.container.ptr + o.via.container.size); + p < pend; ++p) { + s << ", " << *p; } - } else if(std::numeric_limits::is_integer) { - if( static_cast( std::numeric_limits::max()) < x) { return 1; } - if( static_cast(-std::numeric_limits::max()) > x) { return -1; } } - return 0; - } -}; + s << "]"; + break; + // FIXME loop optimiziation -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) <= sizeof(X) ) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + case type::MAP: + s << "{"; + if(o.via.container.size != 0) { + object* p(o.via.container.ptr); + object* const pend(o.via.container.ptr + o.via.container.size*2); + s << *p; ++p; + s << "=>"; + s << *p; ++p; + while(p < pend) { + s << ", "; + s << *p; ++p; + s << "=>"; + s << *p; ++p; } - } else if(std::numeric_limits::is_integer) { - if( static_cast( std::numeric_limits::max()) < x) { return 1; } } - return 0; - } -}; + s << "}"; + break; + // FIXME loop optimiziation -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( static_cast(0) > x ) { return -1; } - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) < sizeof(X) ) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } - } - } else if(std::numeric_limits::is_integer) { - if( static_cast( std::numeric_limits::max()) < x) { return 1; } + default: + // FIXME + s << "#" << std::endl; + } + return s; +} + + +bool operator==(const object x, const object y) +{ + if(x.type != y.type) { return false; } + + switch(x.type) { + case type::NIL: + return true; + + case type::BOOLEAN: + return x.via.boolean == y.via.boolean; + + case type::POSITIVE_INTEGER: + return x.via.u64 == y.via.u64; + + case type::NEGATIVE_INTEGER: + return x.via.i64 == y.via.i64; + + case type::RAW: + return x.via.ref.size == y.via.ref.size && + memcmp(x.via.ref.ptr, y.via.ref.ptr, x.via.ref.size) == 0; + + case type::ARRAY: + if(x.via.container.size != y.via.container.size) { return false; } + for(object* px(x.via.container.ptr), + * const pxend(x.via.container.ptr + x.via.container.size), + * py(y.via.container.ptr); + px != pxend; ++px, ++py) { + if(*px != *py) { return false; } } - return 0; - } -}; + return true; + // FIXME loop optimiziation -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) < sizeof(X) ) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } - } - } else if(std::numeric_limits::is_integer) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + case type::MAP: + if(x.via.container.size != y.via.container.size) { return false; } + for(object* px(x.via.container.ptr), + * const pxend(x.via.container.ptr + x.via.container.size*2), + * py(y.via.container.ptr); + px != pxend; ++px, ++py) { + if(*px != *py) { return false; } } - return 0; + return true; + + default: + return false; } -}; - -template -struct numeric_overflow { - static int test(X x) { - return numeric_overflow_signed_impl::is_signed, std::numeric_limits::is_signed>::test(x); - } - static void check(X x) { - int r = test(x); - if(r == 1) { throw positive_overflow_error(); } - if(r == -1) { throw negative_overflow_error(); } - } -}; - -template -struct numeric_underflow { - static bool test(X x) { - return static_cast(static_cast(x)) != x; - } - static void check(X x) { - if(test(x)) { throw underflow_error(); } - } -}; - -template -inline T integer_cast(X x) { - numeric_overflow::check(x); - return static_cast(x); } - -template -inline T float_cast(X x) { - numeric_overflow::check(x); - numeric_underflow::check(x); - return static_cast(x); } - -template -inline bool numequal(V v, const object_class* x) - try { return v == static_cast(*x); } - catch (type_error&) { return false; } - -template -inline bool numless(V v, const object_class* x) - try { return v < static_cast(*x); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } - -template -inline bool numgreater(V v, const object_class* x) - try { return v > static_cast(*x); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } - -template -inline void numeric_inspect(V v, std::ostream& s) - { s << v; } - -template <> -inline void numeric_inspect(uint8_t v, std::ostream& s) - { s << (uint16_t)v; } - -template <> -inline void numeric_inspect(int8_t v, std::ostream& s) - { s << (int16_t)v; } - -template -inline void numeric_pack(dynamic_packer& p, V v); - -template <> -inline void numeric_pack(dynamic_packer& p, uint8_t v) - { p.pack_uint8(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, uint16_t v) - { p.pack_uint16(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, uint32_t v) - { p.pack_uint32(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, uint64_t v) - { p.pack_uint64(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, int8_t v) - { p.pack_int8(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, int16_t v) - { p.pack_int16(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, int32_t v) - { p.pack_int32(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, int64_t v) - { p.pack_int64(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, float v) - { p.pack_float(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, double v) - { p.pack_double(v); } - -} // noname namespace - - -bool object_nil::isnil() const { return true; } -bool object_nil::operator== (const object_class* x) const - { return typeid(*this) == typeid(*x); } -void object_nil::pack(dynamic_packer& p) const - { p.pack_nil(); } -const object_class* object_nil::inspect(std::ostream& s) const - { s << "nil"; return this; } - -bool object_true::xbool() const { return true; } -bool object_true::operator== (const object_class* x) const - { return typeid(*this) == typeid(*x); } -void object_true::pack(dynamic_packer& p) const - { p.pack_true(); } -const object_class* object_true::inspect(std::ostream& s) const - { s << "true"; return this; } - -bool object_false::xbool() const { return false; } -bool object_false::operator== (const object_class* x) const - { return typeid(*this) == typeid(*x); } -void object_false::pack(dynamic_packer& p) const - { p.pack_false(); } -const object_class* object_false::inspect(std::ostream& s) const - { s << "false"; return this; } - - -#define INTEGER_OBJECT(NAME) \ -uint8_t object_##NAME::xu8 () const { return val; } \ -uint16_t object_##NAME::xu16 () const { return integer_cast(val); } \ -uint32_t object_##NAME::xu32 () const { return integer_cast(val); } \ -uint64_t object_##NAME::xu64 () const { return integer_cast(val); } \ -int8_t object_##NAME::xi8 () const { return integer_cast(val); } \ -int16_t object_##NAME::xi16 () const { return integer_cast(val); } \ -int32_t object_##NAME::xi32 () const { return integer_cast(val); } \ -int64_t object_##NAME::xi64 () const { return integer_cast(val); } \ -float object_##NAME::xfloat () const { return integer_cast(val); } \ -double object_##NAME::xdouble() const { return integer_cast(val); } \ -bool object_##NAME::operator== (const object_class* x) const \ - try { return val == x->x##NAME(); } \ - catch (type_error&) { return false; } \ -bool object_##NAME::operator< (const object_class* x) const \ - try { return val < x->x##NAME(); } \ - catch (positive_overflow_error&) { return true; } \ - catch (overflow_error&) { return false; } \ -bool object_##NAME::operator> (const object_class* x) const \ - try { return val > x->x##NAME(); } \ - catch (negative_overflow_error&) { return true; } \ - catch (overflow_error&) { return false; } \ -void object_##NAME::pack(dynamic_packer& p) const \ - { numeric_pack(p, val); } \ -const object_class* object_##NAME::inspect(std::ostream& s) const \ - { numeric_inspect(val, s); return this; } \ - - -INTEGER_OBJECT(u8) -INTEGER_OBJECT(u16) -INTEGER_OBJECT(u32) -INTEGER_OBJECT(u64) -INTEGER_OBJECT(i8) -INTEGER_OBJECT(i16) -INTEGER_OBJECT(i32) -INTEGER_OBJECT(i64) - -#undef INTEGER_OBJECT - - -#define FLOAT_OBJECT(NAME) \ -uint8_t object_##NAME::xu8 () const { return val; } \ -uint16_t object_##NAME::xu16 () const { return integer_cast(val); } \ -uint32_t object_##NAME::xu32 () const { return integer_cast(val); } \ -uint64_t object_##NAME::xu64 () const { return integer_cast(val); } \ -int8_t object_##NAME::xi8 () const { return integer_cast(val); } \ -int16_t object_##NAME::xi16 () const { return integer_cast(val); } \ -int32_t object_##NAME::xi32 () const { return integer_cast(val); } \ -int64_t object_##NAME::xi64 () const { return integer_cast(val); } \ -float object_##NAME::xfloat () const { return float_cast(val); } \ -double object_##NAME::xdouble() const { return float_cast(val); } \ -bool object_##NAME::operator== (const object_class* x) const \ - try { return val == x->x##NAME(); } \ - catch (type_error&) { return false; } \ -bool object_##NAME::operator< (const object_class* x) const { \ - try { return val < x->xdouble(); } \ - catch (positive_overflow_error&) { return true; } \ - catch (overflow_error&) { return false; } \ - catch (underflow_error&) { \ - if(val < 0.0) { \ - if(numeric_overflow::test(val) == -1) { return true; } \ - try { return static_cast(val) < x->xi64(); } \ - catch (type_error&) { return true; } \ - } else { \ - if(numeric_overflow::test(val) == 1) { return false; } \ - try { return static_cast(val) < x->xu64(); } \ - catch (type_error&) { return false; } \ - } \ - } } \ -bool object_##NAME::operator> (const object_class* x) const { \ - try { return val > x->xdouble(); } \ - catch (negative_overflow_error&) { return true; } \ - catch (overflow_error&) { return false; } \ - catch (underflow_error&) { \ - if(val < 0.0) { \ - if(numeric_overflow::test(val) == -1) { return false; } \ - try { return static_cast(val) > x->xi64(); } \ - catch (type_error&) { return false; } \ - } else { \ - if(numeric_overflow::test(val) == 1) { return true; } \ - try { return static_cast(val) > x->xu64(); } \ - catch (type_error&) { return true; } \ - } \ - } } \ -void object_##NAME::pack(dynamic_packer& p) const \ - { numeric_pack(p, val); } \ -const object_class* object_##NAME::inspect(std::ostream& s) const \ - { s << val; return this; } \ - -FLOAT_OBJECT(float) -FLOAT_OBJECT(double) - -#undef FLOAT_OBJECT - - -#define RAW_OBJECT(NAME, EXTRA) \ -EXTRA \ -bool object_##NAME::operator== (const object_class* x) const \ - try { \ - raw xr(x->xraw()); \ - return len == xr.len && (ptr == xr.ptr || memcmp(ptr, xr.ptr, len) == 0); \ - } catch (type_error&) { return false; } \ -bool object_##NAME::operator< (const object_class* x) const { \ - raw xr(x->xraw()); \ - if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) < 0; } \ - else { return len < xr.len; } } \ -bool object_##NAME::operator> (const object_class* x) const { \ - raw xr(x->xraw()); \ - if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } \ - else { return len > xr.len; } } \ -void object_##NAME::pack(dynamic_packer& p) const \ - { p.pack_raw(ptr, len); } \ -const object_class* object_##NAME::inspect(std::ostream& s) const \ - { (s << '"').write(ptr, len) << '"'; return this; } // FIXME escape +} // FIXME -RAW_OBJECT(mutable_raw_ref, - /*mutable_raw object_mutable_raw_ref::xraw() { return mutable_raw(ptr, len); }*/ - raw object_mutable_raw_ref::xraw() const { return raw(ptr, len); } ) - -RAW_OBJECT(raw_ref, - raw object_raw_ref::xraw() const { return raw(ptr, len); } ) - -#undef RAW_OBJECT - - - array& object_array::xarray() { return val; } -const array& object_array::xarray() const { return val; } -bool object_array::operator== (const object_class* x) const - try { - const std::vector& xa(x->xarray()); - if(val.size() != xa.size()) { return false; } - for(std::vector::const_iterator iv(val.begin()), iv_end(val.end()), ix(xa.begin()); - iv != iv_end; - ++iv, ++ix) { - if(*iv != *ix) { return false; } - } - return true; - } catch (type_error&) { return false; } -const object_class* object_array::inspect(std::ostream& s) const -{ - s << '['; - if(!val.empty()) { - std::vector::const_iterator it(val.begin()); - s << *it; - ++it; - for(std::vector::const_iterator it_end(val.end()); - it != it_end; - ++it) { - s << ", " << *it; - } - } - s << ']'; - return this; -} -void object_array::pack(dynamic_packer& p) const -{ - p.pack_array(val.size()); - for(std::vector::const_iterator it(val.begin()), it_end(val.end()); - it != it_end; - ++it) { - it->pack(p); - } -} - - - map& object_map::xmap() { return val; } -const map& object_map::xmap() const { return val; } -bool object_map::operator== (const object_class* x) const - try { - const std::map& xm(x->xmap()); - if(val.size() != xm.size()) { return false; } - for(std::map::const_iterator iv(val.begin()), iv_end(val.end()), ix(xm.begin()); - iv != iv_end; - ++iv, ++ix) { - if(iv->first != ix->first || iv->second != ix->first) { return false; } - } - return true; - } catch (type_error&) { return false; } -const object_class* object_map::inspect(std::ostream& s) const -{ - s << '{'; - if(!val.empty()) { - std::map::const_iterator it(val.begin()); - s << it->first << "=>" << it->second; - ++it; - for(std::map::const_iterator it_end(val.end()); - it != it_end; - ++it) { - s << ", " << it->first << "=>" << it->second; - } - } - s << '}'; - return this; -} -void object_map::pack(dynamic_packer& p) const -{ - p.pack_map(val.size()); - for(std::map::const_iterator it(val.begin()), it_end(val.end()); - it != it_end; - ++it) { - it->first.pack(p); - it->second.pack(p); - } -} +//template +//const object& operator>> (const object& v, packer& o); } // namespace msgpack diff --git a/cpp/object.hpp b/cpp/object.hpp index 456210c..7007b90 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -1,5 +1,5 @@ // -// MessagePack for C++ dynamic typed objects +// MessagePack for C++ static resolution routine // // Copyright (C) 2008 FURUHASHI Sadayuki // @@ -18,305 +18,109 @@ #ifndef MSGPACK_OBJECT_HPP__ #define MSGPACK_OBJECT_HPP__ -#include +#include "msgpack/pack.hpp" +#include #include #include #include -#include -#include -#include namespace msgpack { -class type_error : public std::bad_cast { }; -class cast_error : public type_error { }; -class overflow_error : public type_error { }; -class underflow_error : public type_error { }; -class positive_overflow_error : public overflow_error { }; -class negative_overflow_error : public overflow_error { }; - - -struct mutable_raw { - explicit mutable_raw() : ptr(NULL), len(0) {} - explicit mutable_raw(char* p, size_t l) : ptr(p), len(l) {} -public: - char* ptr; - size_t len; -public: - std::string str() { return std::string(ptr, len); } -}; - -struct raw { - explicit raw() : ptr(NULL), len(0) {} - explicit raw(const char* p, size_t l) : ptr(p), len(l) {} - raw(const mutable_raw& m) : ptr(m.ptr), len(m.len) {} -public: - const char* ptr; - size_t len; -public: - std::string str() { return std::string(ptr, len); } -}; - - -struct object; - -typedef std::map map; -typedef std::vector array; - -class dynamic_packer; - - -struct object_class { - virtual ~object_class() {} - virtual bool isnil () const { return false; } - virtual bool xbool () const { throw cast_error(); } - virtual uint8_t xu8 () const { throw cast_error(); } - virtual uint16_t xu16 () const { throw cast_error(); } - virtual uint32_t xu32 () const { throw cast_error(); } - virtual uint64_t xu64 () const { throw cast_error(); } - virtual int8_t xi8 () const { throw cast_error(); } - virtual int16_t xi16 () const { throw cast_error(); } - virtual int32_t xi32 () const { throw cast_error(); } - virtual int64_t xi64 () const { throw cast_error(); } - virtual float xfloat () const { throw cast_error(); } - virtual double xdouble() const { throw cast_error(); } - //virtual mutable_raw xraw () { throw cast_error(); } // FIXME - virtual array& xarray() { throw cast_error(); } - virtual map& xmap () { throw cast_error(); } - virtual raw xraw () const { throw cast_error(); } - virtual const array& xarray() const { throw cast_error(); } - virtual const map& xmap () const { throw cast_error(); } - virtual bool operator== (const object_class* x) const { return false; } - virtual bool operator< (const object_class* x) const { throw cast_error(); } - virtual bool operator> (const object_class* x) const { throw cast_error(); } - bool operator!= (const object_class* x) const { return !(this->operator==(x)); } - virtual void pack(dynamic_packer& p) const = 0; - operator bool() const { return xbool(); } // FIXME !isnil(); - operator uint8_t() const { return xu8(); } - operator uint16_t() const { return xu16(); } - operator uint32_t() const { return xu32(); } - operator uint64_t() const { return xu64(); } - operator int8_t() const { return xi8(); } - operator int16_t() const { return xi16(); } - operator int32_t() const { return xi32(); } - operator int64_t() const { return xi64(); } - operator float() const { return xfloat(); } - operator double() const { return xdouble(); } - //operator mutable_raw() { return xraw(); } // FIXME - operator array&() { return xarray(); } - operator map&() { return xmap(); } - operator raw() const { return xraw(); } - operator const array&() const { return xarray(); } - operator const map&() const { return xmap(); } - virtual const object_class* inspect(std::ostream& s) const - { s << '<' << typeid(*this).name() << '>'; return this; } -}; - -inline std::ostream& operator<< (std::ostream& s, const object_class* o) - { o->inspect(s); return s; } - - -struct object_container_mixin {}; -struct object_constructor_mixin {}; +class type_error : public std::bad_cast { }; struct object { - explicit object() : val(NULL) {} - object(object_class* v) : val(v) {} - //object(object_class& v) : val(&v) {} - ~object() {} - bool isnil () const { return val->isnil(); } - bool xbool () const { return val->xbool(); } - uint8_t xu8 () const { return val->xu8(); } - uint16_t xu16 () const { return val->xu16(); } - uint32_t xu32 () const { return val->xu32(); } - uint64_t xu64 () const { return val->xu64(); } - int8_t xi8 () const { return val->xi8(); } - int16_t xi16 () const { return val->xi16(); } - int32_t xi32 () const { return val->xi32(); } - int64_t xi64 () const { return val->xi64(); } - float xfloat () const { return val->xfloat(); } - double xdouble() const { return val->xdouble(); } - //mutable_raw xraw () { return val->xraw(); } // FIXME - array& xarray() { return val->xarray(); } - map& xmap () { return val->xmap(); } - raw xraw () const { return const_cast(val)->xraw(); } - const array& xarray() const { return const_cast(val)->xarray(); } - const map& xmap () const { return const_cast(val)->xmap(); } - bool operator== (object x) const { return val->operator== (x.val); } - bool operator!= (object x) const { return val->operator!= (x.val); } - bool operator< (object x) const { return val->operator< (x.val); } - bool operator> (object x) const { return val->operator> (x.val); } - void pack(dynamic_packer& p) const { val->pack(p); } - template - void pack(Stream& s) const { dynamic_packer p(s); pack(p); } - operator bool() const { return val->operator bool(); } - operator uint8_t() const { return val->operator uint8_t(); } - operator uint16_t() const { return val->operator uint16_t(); } - operator uint32_t() const { return val->operator uint32_t(); } - operator uint64_t() const { return val->operator uint64_t(); } - operator int8_t() const { return val->operator int8_t(); } - operator int16_t() const { return val->operator int16_t(); } - operator int32_t() const { return val->operator int32_t(); } - operator int64_t() const { return val->operator int64_t(); } - operator float() const { return val->operator float(); } - operator double() const { return val->operator double(); } - //operator mutable_raw() { return val->operator mutable_raw(); } // FIXME - operator array&() { return val->operator array&(); } - operator map&() { return val->operator map&(); } - operator raw() const { return val->operator raw(); } - operator const array&() const { return val->operator const array&(); } - operator const map&() const { return val->operator const map&(); } - const object& inspect(std::ostream& s) const - { val->inspect(s); return *this; } -private: - friend class object_container_mixin; - friend class object_constructor_mixin; - object_class* val; + unsigned char type; + union { + bool boolean; + uint64_t u64; + int64_t i64; + double dec; + struct { + object* ptr; + uint32_t size; + } container; + struct { + const char* ptr; + uint32_t size; + } ref; + } via; + // FIXME template operator T() { T v; convert(*this, v); return v; }; }; -inline std::ostream& operator<< (std::ostream& s, const object& o) - { o.inspect(s); return s; } +std::ostream& operator<< (std::ostream& s, const object o); + +bool operator==(const object x, const object y); +inline bool operator!=(const object x, const object y) { return !(x == y); } -struct object_nil : object_class { - bool isnil() const; - bool operator== (const object_class* x) const; - void pack(dynamic_packer& p) const; - const object_class* inspect(std::ostream& s) const; -}; - -struct object_true : object_class { - bool xbool() const; - bool operator== (const object_class* x) const; - void pack(dynamic_packer& p) const; - const object_class* inspect(std::ostream& s) const; -}; - -struct object_false : object_class { - bool xbool() const; - bool operator== (const object_class* x) const; - void pack(dynamic_packer& p) const; - const object_class* inspect(std::ostream& s) const; -}; - -#define INTEGER_CLASS(TYPE, NAME) \ -struct object_##NAME : object_class { \ - explicit object_##NAME(TYPE v) : val(v) {} \ - uint8_t xu8 () const; \ - uint16_t xu16 () const; \ - uint32_t xu32 () const; \ - uint64_t xu64 () const; \ - int8_t xi8 () const; \ - int16_t xi16 () const; \ - int32_t xi32 () const; \ - int64_t xi64 () const; \ - float xfloat () const; \ - double xdouble() const; \ - bool operator== (const object_class* x) const; \ - bool operator< (const object_class* x) const; \ - bool operator> (const object_class* x) const; \ - void pack(dynamic_packer& p) const; \ - const object_class* inspect(std::ostream& s) const; \ -private: \ - TYPE val; \ -}; - -INTEGER_CLASS(uint8_t, u8) -INTEGER_CLASS(uint16_t, u16) -INTEGER_CLASS(uint32_t, u32) -INTEGER_CLASS(uint64_t, u64) -INTEGER_CLASS(int8_t, i8) -INTEGER_CLASS(int16_t, i16) -INTEGER_CLASS(int32_t, i32) -INTEGER_CLASS(int64_t, i64) - -#undef INTEGER_CLASS +template +const object& operator>> (const object& v, packer& o); -#define FLOAT_CLASS(TYPE, NAME) \ -struct object_##NAME : object_class { \ - object_##NAME(TYPE v) : val(v) {} \ - uint8_t xu8 () const; \ - uint16_t xu16 () const; \ - uint32_t xu32 () const; \ - uint64_t xu64 () const; \ - int8_t xi8 () const; \ - int16_t xi16 () const; \ - int32_t xi32 () const; \ - int64_t xi64 () const; \ - float xfloat () const; \ - double xdouble() const; \ - bool operator== (const object_class* x) const; \ - bool operator< (const object_class* x) const; \ - bool operator> (const object_class* x) const; \ - void pack(dynamic_packer& p) const; \ - const object_class* inspect(std::ostream& s) const; \ -private: \ - TYPE val; \ -}; - -FLOAT_CLASS(float, float) -FLOAT_CLASS(double, double) - -#undef FLOAT_CLASS +namespace type { + static const unsigned char NIL = 0x01; + static const unsigned char BOOLEAN = 0x02; + static const unsigned char POSITIVE_INTEGER = 0x03; + static const unsigned char NEGATIVE_INTEGER = 0x04; + static const unsigned char DOUBLE = 0x05; + static const unsigned char RAW = 0x06; + static const unsigned char ARRAY = 0x07; + static const unsigned char MAP = 0x08; -#define RAW_CLASS(NAME, TYPE, EXTRA) \ -struct object_##NAME : object_class { \ - explicit object_##NAME(TYPE p, uint32_t l) : ptr(p), len(l) {} \ - EXTRA \ - bool operator== (const object_class* x) const; \ - bool operator< (const object_class* x) const; \ - bool operator> (const object_class* x) const; \ - void pack(dynamic_packer& p) const; \ - const object_class* inspect(std::ostream& s) const; \ -private: \ - TYPE ptr; \ - uint32_t len; \ -}; - -// FIXME -RAW_CLASS(mutable_raw_ref, char*, /*mutable_raw xraw();*/ raw xraw() const; ) -RAW_CLASS(raw_ref, const char*, raw xraw() const; ) - -#undef RAW_CLASS + template + inline T& operator<< (T& v, object o) + { + v = o; + return v; + } -struct object_array : object_class, object_container_mixin { - explicit object_array() {} - explicit object_array(uint32_t n) { val.reserve(n); } - array& xarray(); - const array& xarray() const; - bool operator== (const object_class* x) const; - // FIXME operator<, operator> - void pack(dynamic_packer& p) const; - const object_class* inspect(std::ostream& s) const; -public: - void push_back(object o) { val.push_back(o); } -private: - std::vector val; -}; + namespace detail { + template + inline void pack_copy(T v, packer& o) + { pack(v, o); } + } + + template + inline const T& operator>> (const T& v, packer& o) + { + detail::pack_copy(v.pack(), o); + return v; + } + +} // namespace type -// FIXME hash, operator==: nil, true, false, array, mapを入れられãªã„ -struct object_map : object_class, object_container_mixin { - explicit object_map() {} - map& xmap(); - const map& xmap() const; - bool operator== (const object_class* x) const; - // FIXME operator<, operator> - void pack(dynamic_packer& p) const; - const object_class* inspect(std::ostream& s) const; -public: - void store(object k, object v) { val[k] = v; } -private: - std::map val; -}; +template +inline void convert(T& v, object o) +{ + using namespace type; + v << o; +} + + +template +inline void pack(T& v, packer& o) +{ + using namespace type; + v >> o; +} + + +template +inline void pack(T& v, Stream& s) +{ + packer pk(s); + pack(v, pk); +} } // namespace msgpack +#include "msgpack/type.hpp" + #endif /* msgpack/object.hpp */ diff --git a/cpp/pack.hpp b/cpp/pack.hpp index f3eeb34..fe470b6 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -18,8 +18,6 @@ #ifndef MSGPACK_PACK_HPP__ #define MSGPACK_PACK_HPP__ -#include "msgpack/object.hpp" -#include "msgpack/zone.hpp" #include // __BYTE_ORDER #include @@ -44,9 +42,9 @@ public: void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } void pack_float(float d) { pack_float_impl(m_stream, d); } void pack_double(double d) { pack_double_impl(m_stream, d); } - void pack_nil() { pack_nil(m_stream); } - void pack_true() { pack_true(m_stream); } - void pack_false() { pack_false(m_stream); } + void pack_nil() { pack_nil_impl(m_stream); } + void pack_true() { pack_true_impl(m_stream); } + void pack_false() { pack_false_impl(m_stream); } void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } void pack_raw(const char* b, size_t l) { pack_raw_impl(m_stream, (const void*)b, l); } @@ -92,6 +90,7 @@ template packer::packer(Stream& s) : m_stream(s) { } +/* class dynamic_stream { public: template @@ -212,6 +211,7 @@ inline void pack(Stream& s, object o) dynamic_packer pk(s); o.pack(pk); } +*/ } // namespace msgpack diff --git a/cpp/test.cpp b/cpp/test.cpp index 68050a3..dff9101 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -1,37 +1,35 @@ #include #include -//#include -//#include #include #include #include +using namespace msgpack; + class checker { public: - void check(const char* d, size_t len, msgpack::object should) { - using msgpack::object; + template + void check(const char* d, size_t len, T should) { try { std::cout << "----" << std::endl; object o; try { - o = msgpack::unpack(d, len, m_zone); + o = unpack(d, len, m_zone); } catch (std::runtime_error& e) { - std::cout << should << std::endl; + std::cout << o << std::endl; std::cout << "**" << e.what() << "**" << std::endl; return; } std::cout << o << std::endl; - if(o != should) { - std::cout << "** TEST FAILED **" << std::endl; - } try { std::stringstream s; - msgpack::pack(s, o); + pack(should, s); std::string str(s.str()); - object ro = msgpack::unpack(str.data(), str.size(), m_zone); + object ro = unpack(str.data(), str.size(), m_zone); + std::cout << ro << std::endl; if(ro != o) { throw std::runtime_error("NOT MATCH"); } } catch (std::runtime_error& e) { std::cout << "** REUNPACK FAILED **" << std::endl; @@ -45,7 +43,7 @@ public: m_zone.clear(); } private: - msgpack::zone m_zone; + zone m_zone; }; int main(void) @@ -53,54 +51,48 @@ int main(void) checker c; { // SimpleValue - msgpack::zone z; const char d[] = { 0x93, 0xc0, 0xc2, 0xc3, }; c.check(d, sizeof(d), - z.narray( - z.nnil(), z.nfalse(), z.ntrue() + type::make_tuple( + type::nil(), false, true ) ); } { // Fixnum - msgpack::zone z; const char d[] = { 0x92, 0x93, 0x00, 0x40, 0x7f, 0x93, 0xe0, 0xf0, 0xff, }; c.check(d, sizeof(d), - z.narray( - z.narray( - z.nu8(0), - z.nu8(64), - z.nu8(127) + type::make_tuple( + type::make_tuple( + 0, 64, 127 ), - z.narray( - z.ni8(-32), - z.ni8(-16), - z.ni8(-1) + type::make_tuple( + -32, -16, -1 ) ) ); } { // FixArray - msgpack::zone z; const char d[] = { 0x92, 0x90, 0x91, 0x91, 0xc0, }; + std::vector empty; c.check(d, sizeof(d), - z.narray( - z.narray(), - z.narray( - z.narray( - z.nnil() + type::make_tuple( + empty, + type::make_tuple( + type::make_tuple( + type::nil() ) ) ) @@ -108,7 +100,6 @@ int main(void) } { // FixRaw - msgpack::zone z; const char d[] = { 0x94, 0xa0, @@ -117,91 +108,84 @@ int main(void) 0xa3, 'd', 'e', 'f', }; c.check(d, sizeof(d), - z.narray( - z.nraw_ref("", 0), - z.nraw_ref("a", 1), - z.nraw_ref("bc", 2), - z.nraw_ref("def", 3) + type::make_tuple( + std::string(""), + std::string("a"), + std::string("bc"), + type::raw_ref("def", 3) ) ); } - static const uint16_t TASK_ARRAY = 100; - static char tarray[3]; - static char traw[64]; + static const unsigned TASK_ARRAY = 100; + static const unsigned TASK_REPEAT = 10; + std::vector task; + + // create task { + static char traw[64]; memset(traw, 'a', sizeof(traw)); - traw[0] = 0xda; - uint16_t n = htons(sizeof(traw)-3); - traw[1] = ((char*)&n)[0]; - traw[2] = ((char*)&n)[1]; - - msgpack::zone z; - std::cout << msgpack::unpack(traw, sizeof(traw), z) << std::endl;; - } - - { - tarray[0] = 0xdc; - uint16_t n = htons(TASK_ARRAY); - tarray[1] = ((char*)&n)[0]; - tarray[2] = ((char*)&n)[1]; - } - - { - // write message - ssize_t total_bytes = 0; - std::stringstream stream; - for(unsigned q=0; q < 10; ++q) { - stream.write(tarray, sizeof(tarray)); - total_bytes += sizeof(tarray); - for(uint16_t i=0; i < TASK_ARRAY; ++i) { - stream.write(traw, sizeof(traw)); - total_bytes += sizeof(traw); - } + + task.resize(TASK_ARRAY); + for(unsigned i=0; i < TASK_ARRAY; ++i) { + task[i] = std::string(traw, sizeof(traw)); } + } - stream.seekg(0); - // reserive message + std::stringstream stream; + + // send message + { + for(unsigned i=0; i < TASK_REPEAT; ++i) { + pack(task, stream); + } + std::cout << "send " << stream.str().size() << " bytes" << std::endl; + } + + ssize_t total_bytes = stream.str().size(); + stream.seekg(0); + + // reserive message + { unsigned num_msg = 0; - static const size_t RESERVE_SIZE = 32;//*1024; - msgpack::unpacker upk; + std::auto_ptr pz(new zone()); + + unpacker pac(*pz); + while(stream.good() && total_bytes > 0) { // 1. reserve buffer - upk.reserve_buffer(RESERVE_SIZE); + pac.reserve_buffer(RESERVE_SIZE); // 2. read data to buffer() up to buffer_capacity() bytes size_t sz = stream.readsome( - upk.buffer(), - upk.buffer_capacity()); + pac.buffer(), + pac.buffer_capacity()); total_bytes -= sz; std::cout << "read " << sz << " bytes to capacity " - << upk.buffer_capacity() << " bytes" + << pac.buffer_capacity() << " bytes" << std::endl; // 3. specify the number of bytes actually copied - upk.buffer_consumed(sz); + pac.buffer_consumed(sz); // 4. repeat execute() until it returns false - while( upk.execute() ) { - std::cout << "message parsed" << std::endl; - + while( pac.execute() ) { // 5.1. take out the parsed object - msgpack::object o = upk.data(); + object o = pac.data(); - // 5.2. the parsed object is valid until the zone is deleted - std::auto_ptr pz(upk.release_zone()); - - std::cout << o << std::endl; + // do something using pz and o + std::cout << "message parsed: " << o << std::endl; ++num_msg; - // 5.3 re-initialize unpacker - upk.reset(); + // 5.3 re-initialize unpacker with next zone */ + pz.reset(new zone()); + pac.reset(*pz); } } diff --git a/cpp/type.hpp b/cpp/type.hpp new file mode 100644 index 0000000..1dfd414 --- /dev/null +++ b/cpp/type.hpp @@ -0,0 +1,9 @@ +#include "msgpack/type/array.hpp" +#include "msgpack/type/boolean.hpp" +#include "msgpack/type/float.hpp" +#include "msgpack/type/integer.hpp" +#include "msgpack/type/map.hpp" +#include "msgpack/type/nil.hpp" +#include "msgpack/type/raw.hpp" +#include "msgpack/type/tuple.hpp" + diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp new file mode 100644 index 0000000..7066703 --- /dev/null +++ b/cpp/type/array.hpp @@ -0,0 +1,59 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_ARRAY_HPP__ +#define MSGPACK_TYPE_ARRAY_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { +namespace type { + + +template +inline std::vector operator<< (std::vector& v, object o) +{ + if(o.type != ARRAY) { throw type_error(); } + v.resize(o.via.container.size); + object* p(o.via.container.ptr); + object* const pend(o.via.container.ptr + o.via.container.size); + T* it(&v.front()); + for(; p < pend; ++p, ++it) { + convert(*it, *p); + } + return v; +} + + +template +inline const std::vector& operator>> (const std::vector& v, packer& o) +{ + o.pack_array(v.size()); + for(typename std::vector::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + pack(*it, o); + } + return v; +} + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/array.hpp */ + diff --git a/cpp/type/boolean.hpp b/cpp/type/boolean.hpp new file mode 100644 index 0000000..0538495 --- /dev/null +++ b/cpp/type/boolean.hpp @@ -0,0 +1,49 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_BOOLEAN_HPP__ +#define MSGPACK_TYPE_BOOLEAN_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { +namespace type { + + +inline bool& operator<< (bool& v, object o) +{ + if(o.type != BOOLEAN) { throw type_error(); } + v = o.via.boolean; + return v; +} + + +template +inline const bool& operator>> (const bool& v, packer o) +{ + if(v) { o.pack_true(); } + else { o.pack_false(); } + return v; +} + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/bool.hpp */ + diff --git a/cpp/type/float.hpp b/cpp/type/float.hpp new file mode 100644 index 0000000..11fa280 --- /dev/null +++ b/cpp/type/float.hpp @@ -0,0 +1,67 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_FLOAT_HPP__ +#define MSGPACK_TYPE_FLOAT_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { +namespace type { + + +// FIXME check overflow, underflow + + +inline float& operator<< (float& v, object o) +{ + if(o.type != DOUBLE) { throw type_error(); } + v = o.via.dec; + return v; +} + + +template +inline const float& operator>> (const float& v, packer o) +{ + o.pack_float(v); + return v; +} + + +inline double& operator<< (double& v, object o) +{ + if(o.type != DOUBLE) { throw type_error(); } + v = o.via.dec; + return v; +} + + +template +inline const double& operator>> (const double& v, packer o) +{ + o.pack_double(v); + return v; +} + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/float.hpp */ + diff --git a/cpp/type/integer.hpp b/cpp/type/integer.hpp new file mode 100644 index 0000000..488c9d4 --- /dev/null +++ b/cpp/type/integer.hpp @@ -0,0 +1,243 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_INTEGER_HPP__ +#define MSGPACK_TYPE_INTEGER_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { +namespace type { + + +namespace detail { + template + struct convert_integer_sign; + + template + struct convert_integer_sign { + static inline T convert(object o) { + if(o.type == POSITIVE_INTEGER) { + if(o.via.u64 > (uint64_t)std::numeric_limits::max()) + { throw type_error(); } + return o.via.u64; + } else if(o.type == NEGATIVE_INTEGER) { + if(o.via.i64 < (int64_t)-std::numeric_limits::max()) + { throw type_error(); } + return o.via.i64; + } + throw type_error(); + } + }; + + template + struct convert_integer_sign { + static inline T convert(object o) { + if(o.type == POSITIVE_INTEGER) { + if(o.via.u64 > (uint64_t)std::numeric_limits::max()) + { throw type_error(); } + return o.via.u64; + } + throw type_error(); + } + }; + + template + static inline T convert_integer(object o) + { + return detail::convert_integer_sign::is_signed>::convert(o); + } + + + + template + struct pack_integer_size_sign; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) + { o.pack_int8(v); } + }; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) + { o.pack_uint8(v); } + }; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) { + if( (int16_t)v <= (int16_t)std::numeric_limits::max() && + (int16_t)v >= (int16_t)std::numeric_limits::min()) + { o.pack_int8(v); } + else { o.pack_int16(v); } + } + }; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) { + if( (uint16_t)v <= (uint16_t)std::numeric_limits::max()) + { o.pack_uint8(v); } + else { o.pack_uint16(v); } + } + }; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) { + if( (int32_t)v <= (int32_t)std::numeric_limits::max() && + (int32_t)v >= (int32_t)std::numeric_limits::min()) + { o.pack_int8(v); } + else if( (int32_t)v <= (int32_t)std::numeric_limits::max() && + (int32_t)v >= (int32_t)std::numeric_limits::min()) + { o.pack_int16(v); } + else { o.pack_int32(v); } + } + }; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) { + if( (uint32_t)v <= (uint32_t)std::numeric_limits::max()) + { o.pack_uint8(v); } + else if( (uint32_t)v <= (uint32_t)std::numeric_limits::max()) + { o.pack_uint16(v); } + else { o.pack_uint32(v); } + } + }; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) { + if( (int64_t)v <= (int64_t)std::numeric_limits::max() && + (int64_t)v >= (int64_t)std::numeric_limits::min()) + { o.pack_int8(v); } + else if( (int64_t)v <= (int64_t)std::numeric_limits::max() && + (int64_t)v >= (int64_t)std::numeric_limits::min()) + { o.pack_int16(v); } + else if( (int64_t)v <= (int64_t)std::numeric_limits::max() && + (int64_t)v >= (int64_t)std::numeric_limits::min()) + { o.pack_int32(v); } + else { o.pack_int64(v); } + } + }; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) { + if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) + { o.pack_uint8(v); } + else if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) + { o.pack_uint16(v); } + else if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) + { o.pack_uint32(v); } + else { o.pack_uint64(v); } + } + }; + + + template + static inline void pack_integer(T v, packer& o) + { + pack_integer_size_sign::is_signed>::pack(v, o); + } + +} // namespace detail + + +inline signed char& operator<< (signed char& v, object o) + { v = detail::convert_integer(o); return v; } + +inline signed short& operator<< (signed short& v, object o) + { v = detail::convert_integer(o); return v; } + +inline signed int& operator<< (signed int& v, object o) + { v = detail::convert_integer(o); return v; } + +inline signed long& operator<< (signed long& v, object o) + { v = detail::convert_integer(o); return v; } + +inline signed long long& operator<< (signed long long& v, object o) + { v = detail::convert_integer(o); return v; } + + +inline unsigned char& operator<< (unsigned char& v, object o) + { v = detail::convert_integer(o); return v; } + +inline unsigned short& operator<< (unsigned short& v, object o) + { v = detail::convert_integer(o); return v; } + +inline unsigned int& operator<< (unsigned int& v, object o) + { v = detail::convert_integer(o); return v; } + +inline unsigned long& operator<< (unsigned long& v, object o) + { v = detail::convert_integer(o); return v; } + +inline unsigned long long& operator<< (unsigned long long& v, object o) + { v = detail::convert_integer(o); return v; } + + +template +inline const signed char& operator>> (const signed char& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const signed short& operator>> (const signed short& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const signed int& operator>> (const signed int& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const signed long& operator>> (const signed long& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const signed long long& operator>> (const signed long long& v, packer o) + { detail::pack_integer(v, o); return v; } + + +template +inline const unsigned char& operator>> (const unsigned char& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const unsigned short& operator>> (const unsigned short& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const unsigned int& operator>> (const unsigned int& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const unsigned long& operator>> (const unsigned long& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const unsigned long long& operator>> (const unsigned long long& v, packer o) + { detail::pack_integer(v, o); return v; } + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/integer.hpp */ + diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp new file mode 100644 index 0000000..1efecec --- /dev/null +++ b/cpp/type/map.hpp @@ -0,0 +1,132 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_MAP_HPP__ +#define MSGPACK_TYPE_MAP_HPP__ + +#include "msgpack/object.hpp" +#include +#include + +namespace msgpack { +namespace type { + + +template +class assoc_vector : public std::vector< std::pair > {}; + +namespace detail { + template + struct pair_first_less { + bool operator() (const std::pair& x, const std::pair& y) const + { return x.first < y.first; } + }; +} + +template +inline assoc_vector& operator<< (assoc_vector& v, object o) +{ + if(o.type != MAP) { throw type_error(); } + v.resize(o.via.container.size); + object* p(o.via.container.ptr); + object* const pend(o.via.container.ptr + o.via.container.size); + std::pair* it(&v.front()); + for(; p < pend; ++it) { + convert(it->first, *p); ++p; + convert(it->second, *p); ++p; + } + std::sort(v.begin(), v.end(), detail::pair_first_less()); + return v; +} + +template +inline const assoc_vector& operator>> (const assoc_vector& v, packer& o) +{ + o.pack_map(v.size()); + for(typename assoc_vector::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + pack(it->first, o); + pack(it->second, o); + } +} + + +template +inline std::map operator<< (std::map& v, object o) +{ + if(o.type != MAP) { throw type_error(); } + object* p(o.via.container.ptr); + object* const pend(o.via.container.ptr + o.via.container.size*2); + while(p < pend) { + K key; + convert(key, *p); ++p; + typename std::map::iterator it(v.find(key)); + if(it != v.end()) { + V val; + convert(val, *p); ++p; + it->insert( std::pair(key, val) ); + } else { + convert(it->second, *p); ++p; + } + } + return v; +} + +template +inline const std::map& operator>> (const std::map& v, packer& o) +{ + o.pack_map(v.size()); + for(typename std::map::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + pack(it->first, o); + pack(it->second, o); + } +} + + +template +inline std::multimap operator<< (std::multimap& v, object o) +{ + if(o.type != MAP) { throw type_error(); } + object* p(o.via.container.ptr); + object* const pend(o.via.container.ptr + o.via.container.size*2); + while(p < pend) { + std::pair value; + convert(value.first, *p); ++p; + convert(value.second, *p); ++p; + v.insert(value); + } + return v; +} + +template +inline const std::multimap& operator>> (const std::multimap& v, packer& o) +{ + o.pack_multimap(v.size()); + for(typename std::multimap::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + pack(it->first, o); + pack(it->second, o); + } +} + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/map.hpp */ + diff --git a/cpp/type/nil.hpp b/cpp/type/nil.hpp new file mode 100644 index 0000000..73cedb4 --- /dev/null +++ b/cpp/type/nil.hpp @@ -0,0 +1,47 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_NIL_HPP__ +#define MSGPACK_TYPE_NIL_HPP__ + +#include "msgpack/object.hpp" + +namespace msgpack { +namespace type { + + +struct nil { }; + +inline nil& operator<< (nil& v, object o) +{ + if(o.type != NIL) { throw type_error(); } + return v; +} + +template +inline const nil& operator>> (const nil& v, packer& o) +{ + o.pack_nil(); + return v; +} + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/nil.hpp */ + diff --git a/cpp/type/raw.hpp b/cpp/type/raw.hpp new file mode 100644 index 0000000..8767238 --- /dev/null +++ b/cpp/type/raw.hpp @@ -0,0 +1,99 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_RAW_HPP__ +#define MSGPACK_TYPE_RAW_HPP__ + +#include "msgpack/object.hpp" +#include +#include + +namespace msgpack { +namespace type { + + +struct raw_ref { + raw_ref() : ptr(NULL), size(0) {} + raw_ref(const char* p, uint32_t s) : ptr(p), size(s) {} + + const char* ptr; + uint32_t size; + + std::string str() { return std::string(ptr, size); } + + bool operator== (const raw_ref& x) + { + return size == x.size && memcmp(ptr, x.ptr, size) == 0; + } + + bool operator!= (const raw_ref& x) + { + return !(*this != x); + } + + bool operator< (const raw_ref& x) + { + if(size == x.size) { return memcmp(ptr, x.ptr, size) < 0; } + else { return size < x.size; } + } + + bool operator> (const raw_ref& x) + { + if(size == x.size) { return memcmp(ptr, x.ptr, size) > 0; } + else { return size > x.size; } + } +}; + +inline raw_ref& operator<< (raw_ref& v, object o) +{ + if(o.type != RAW) { throw type_error(); } + v.ptr = o.via.ref.ptr; + v.size = o.via.ref.size; + return v; +} + + +inline std::string& operator<< (std::string& v, object o) +{ + raw_ref r; + r << o; + v.assign(r.ptr, r.size); + return v; +} + + +template +inline const raw_ref& operator>> (const raw_ref& v, packer& o) +{ + o.pack_raw(v.ptr, v.size); + return v; +} + + +template +inline const std::string& operator>> (const std::string& v, packer& o) +{ + o.pack_raw(v.data(), v.size()); + return v; +} + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/raw.hpp */ + diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb new file mode 100644 index 0000000..2a8a2dd --- /dev/null +++ b/cpp/type/tuple.hpp.erb @@ -0,0 +1,172 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_TUPLE_HPP__ +#define MSGPACK_TYPE_TUPLE_HPP__ + +#include "msgpack/object.hpp" + +namespace msgpack { +namespace type { + + +// FIXME operator== +// FIXME operator!= + +<% GENERATION_LIMIT = 15 %> + +template < typename A0 <%1.upto(GENERATION_LIMIT+1) {|i|%>, typename A<%=i%> = void<%}%> > +struct tuple; + +template +struct tuple_type; + +template +struct const_tuple_type; + +template +struct tuple_element { + typedef T type; + tuple_element(T& x) : _x(x) {} + type& get() { return _x; } + const type& get() const { return _x; } +private: + type& _x; +}; + +template +struct const_tuple_element { + typedef T type; + const_tuple_element(const T& x) : _x(x) {} + const type& get() const { return _x; } +private: + const type& _x; +}; + + +<%0.upto(GENERATION_LIMIT) {|i|%> +<%0.upto(i) {|j|%> +template < typename A0 <%1.upto(i) {|k|%>, typename A<%=k%> <%}%>> +struct tuple_type, A<%=k%> <%}%>>, <%=j%>> : tuple_element> { + tuple_type(tuple, A<%=k%> <%}%>>& x) : tuple_element>(x.a<%=j%>) {} +}; +<%}%> +<%}%> + + +<%0.upto(GENERATION_LIMIT) {|i|%> +<%0.upto(i) {|j|%> +template < typename A0 <%1.upto(i) {|k|%>, typename A<%=k%> <%}%>> +struct const_tuple_type, A<%=k%> <%}%>>, <%=j%>> : const_tuple_element> { + const_tuple_type(const tuple, A<%=k%> <%}%>>& x) : const_tuple_element>(x.a<%=j%>) {} +}; +<%}%> +<%}%> + + +<%0.upto(GENERATION_LIMIT) {|i|%> +template < typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> +tuple< A0 <%1.upto(i) {|j|%>, A<%=j%> <%}%>> make_tuple(const A0& a0 <%1.upto(i) {|j|%>, const A<%=j%>& a<%=j%><%}%>) +{ + return tuple< A0 <%1.upto(i) {|j|%>, A<%=j%> <%}%>>(a0 <%1.upto(i) {|j|%>, a<%=j%><%}%>); +} +<%}%> + + +<%0.upto(GENERATION_LIMIT) {|i|%> +template < typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> +struct tuple, A<%=j%> <%}%>> { + tuple() {} + tuple(const A0& _a0 <%1.upto(i) {|j|%>, const A<%=j%>& _a<%=j%><%}%>) : + a0(_a0) <%1.upto(i) {|j|%>, a<%=j%>(_a<%=j%>)<%}%> {} + tuple(object o) { convert(*this, o); } + template typename tuple_type, A<%=j%> <%}%>>, N>::type& get() + { return tuple_type, A<%=j%> <%}%>>, N>(*this).get(); } + template const typename const_tuple_type, A<%=j%> <%}%>>, N>::type& get() const + { return const_tuple_type, A<%=j%> <%}%>>, N>(*this).get(); } + <%0.upto(i) {|j|%> + A<%=j%> a<%=j%>;<%}%> +}; +<%}%> + + +<%0.upto(GENERATION_LIMIT) {|i|%> +template < typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> +tuple, A<%=j%> <%}%>>& operator<< ( + tuple, A<%=j%> <%}%>>& v, + object o) { + if(o.type != ARRAY) { throw type_error(); } + if(o.via.container.size < <%=i+1%>) { throw type_error(); } + <%0.upto(i) {|j|%> + convert>(v.template get<<%=j%>>(), o.via.container.ptr[<%=j%>]);<%}%> + return v; +} +<%}%> + + +// FIXME +/* +template +struct tuple_just; + +template +struct tuple_just { + A0 a0; + static inline void convert(object o, tuple_just& v) + { + if(o.type != ARRAY) { throw type_error(); } + if(o.v.container.size != 1) { throw type_error(); } + msgpack::convert(o.v.container.ptr[0], v.a0); + } +}; + +template +struct tuple_just { + A0 a0; + A1 a1; + static inline void convert(object o, tuple_just& v) + { + if(o.type != ARRAY) { throw type_error(); } + if(o.v.container.size != 2) { throw type_error(); } + msgpack::convert(o.v.container.ptr[0], v.a0); + msgpack::convert(o.v.container.ptr[1], v.a1); + } +}; +*/ + + + +<%0.upto(GENERATION_LIMIT) {|i|%> +template < typename Stream , typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> +const tuple, A<%=j%> <%}%>>& operator>> ( + const tuple, A<%=j%> <%}%>>& v, + packer o) { + o.pack_array(<%=i+1%>); + <%0.upto(i) {|j|%> + pack(v.template get<<%=j%>>(), o);<%}%> + return v; +} +<%}%> + + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/tuple.hpp */ + + diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 873d3da..71593a0 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -31,7 +31,7 @@ namespace msgpack { #define msgpack_unpack_callback(name) \ msgpack_unpack_##name -#define msgpack_unpack_object object_class* +#define msgpack_unpack_object object #define msgpack_unpack_user zone* @@ -40,68 +40,88 @@ struct msgpack_unpacker_context; static void msgpack_unpacker_init(struct msgpack_unpacker_context* ctx); -static object_class* msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); +static object msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, const char* data, size_t len, size_t* off); -static inline object_class* msgpack_unpack_init(zone** z) -{ return NULL; } +static inline object msgpack_unpack_init(zone** z) +{ return object(); } -static inline object_class* msgpack_unpack_uint8(zone** z, uint8_t d) -{ return (*z)->nu8(d); } +static inline object msgpack_unpack_uint8(zone** z, uint8_t d) +{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object_class* msgpack_unpack_uint16(zone** z, uint16_t d) -{ return (*z)->nu16(d); } +static inline object msgpack_unpack_uint16(zone** z, uint16_t d) +{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object_class* msgpack_unpack_uint32(zone** z, uint32_t d) -{ return (*z)->nu32(d); } +static inline object msgpack_unpack_uint32(zone** z, uint32_t d) +{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object_class* msgpack_unpack_uint64(zone** z, uint64_t d) -{ return (*z)->nu64(d); } +static inline object msgpack_unpack_uint64(zone** z, uint64_t d) +{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object_class* msgpack_unpack_int8(zone** z, int8_t d) -{ return (*z)->ni8(d); } +static inline object msgpack_unpack_int8(zone** z, int8_t d) +{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object_class* msgpack_unpack_int16(zone** z, int16_t d) -{ return (*z)->ni16(d); } +static inline object msgpack_unpack_int16(zone** z, int16_t d) +{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object_class* msgpack_unpack_int32(zone** z, int32_t d) -{ return (*z)->ni32(d); } +static inline object msgpack_unpack_int32(zone** z, int32_t d) +{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object_class* msgpack_unpack_int64(zone** z, int64_t d) -{ return (*z)->ni64(d); } +static inline object msgpack_unpack_int64(zone** z, int64_t d) +{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object_class* msgpack_unpack_float(zone** z, float d) -{ return (*z)->nfloat(d); } +static inline object msgpack_unpack_float(zone** z, float d) +{ object o; o.type = type::DOUBLE; o.via.dec = d; return o; } -static inline object_class* msgpack_unpack_double(zone** z, double d) -{ return (*z)->ndouble(d); } +static inline object msgpack_unpack_double(zone** z, double d) +{ object o; o.type = type::DOUBLE; o.via.dec = d; return o; } -static inline object_class* msgpack_unpack_nil(zone** z) -{ return (*z)->nnil(); } +static inline object msgpack_unpack_nil(zone** z) +{ object o; o.type = type::NIL; return o; } -static inline object_class* msgpack_unpack_true(zone** z) -{ return (*z)->ntrue(); } +static inline object msgpack_unpack_true(zone** z) +{ object o; o.type = type::BOOLEAN; o.via.boolean = true; return o; } -static inline object_class* msgpack_unpack_false(zone** z) -{ return (*z)->nfalse(); } +static inline object msgpack_unpack_false(zone** z) +{ object o; o.type = type::BOOLEAN; o.via.boolean = false; return o; } -static inline object_class* msgpack_unpack_array(zone** z, unsigned int n) -{ return (*z)->narray(n); } +static inline object msgpack_unpack_array(zone** z, unsigned int n) +{ + object o; + o.type = type::ARRAY; + o.via.container.size = 0; + o.via.container.ptr = (*z)->malloc_container(n); + return o; +} -static inline void msgpack_unpack_array_item(zone** z, object_class* c, object_class* o) -{ reinterpret_cast(c)->push_back(o); } +static inline void msgpack_unpack_array_item(zone** z, object* c, object o) +{ c->via.container.ptr[ c->via.container.size++ ] = o; } -static inline object_class* msgpack_unpack_map(zone** z, unsigned int n) -{ return (*z)->nmap(); } +static inline object msgpack_unpack_map(zone** z, unsigned int n) +{ + object o; + o.type = type::MAP; + o.via.container.size = 0; + o.via.container.ptr = (*z)->malloc_container(n*2); + return o; +} -static inline void msgpack_unpack_map_item(zone** z, object_class* c, object_class* k, object_class* v) -{ reinterpret_cast(c)->store(k, v); } +static inline void msgpack_unpack_map_item(zone** z, object* c, object k, object v) +{ + c->via.container.ptr[ c->via.container.size ] = k; + c->via.container.ptr[ c->via.container.size+1 ] = v; + ++c->via.container.size; +} -static inline object_class* msgpack_unpack_raw(zone** z, const char* b, const char* p, unsigned int l) -{ return (*z)->nraw_ref(p, l); } +static inline object msgpack_unpack_raw(zone** z, const char* b, const char* p, unsigned int l) +{ object o; o.type = type::RAW; o.via.ref.ptr = p; o.via.ref.size = l; return o; } #include "msgpack/unpack_template.h" @@ -121,7 +141,7 @@ struct unpacker::context { return msgpack_unpacker_execute(&m_ctx, data, len, off); } - object_class* data() + object data() { return msgpack_unpacker_data(&m_ctx); } @@ -158,9 +178,8 @@ private: }; -unpacker::unpacker() : - m_zone(new zone()), - m_ctx(new context(m_zone)), +unpacker::unpacker(zone& z) : + m_ctx(new context(&z)), m_buffer(NULL), m_used(0), m_free(0), @@ -170,9 +189,7 @@ unpacker::unpacker() : unpacker::~unpacker() { - free(m_buffer); delete m_ctx; - delete m_zone; } @@ -184,32 +201,16 @@ void unpacker::expand_buffer(size_t len) else { next_size = UNPACKER_INITIAL_BUFFER_SIZE; } while(next_size < len + m_used) { next_size *= 2; } - char* tmp = (char*)realloc(m_buffer, next_size); - if(!tmp) { throw std::bad_alloc(); } - m_buffer = tmp; - //char* tmp = (char*)malloc(next_size); - //if(!tmp) { throw std::bad_alloc(); } - //memcpy(tmp, m_buffer, m_used); - //free(m_buffer); - - m_buffer = tmp; + m_buffer = m_ctx->user()->realloc(m_buffer, next_size); m_free = next_size - m_used; } else { size_t next_size = UNPACKER_INITIAL_BUFFER_SIZE; while(next_size < len + m_used - m_off) { next_size *= 2; } - char* tmp = (char*)malloc(next_size); - if(!tmp) { throw std::bad_alloc(); } + char* tmp = m_ctx->user()->malloc(next_size); memcpy(tmp, m_buffer+m_off, m_used-m_off); - try { - m_zone->push_finalizer(&zone::finalize_free, NULL, m_buffer); - } catch (...) { - free(tmp); - throw; - } - m_buffer = tmp; m_used = m_used - m_off; m_free = next_size - m_used; @@ -230,29 +231,15 @@ bool unpacker::execute() } } -zone* unpacker::release_zone() -{ - zone* nz = new zone(); - zone* z = m_zone; - m_zone = nz; - m_ctx->user(m_zone); - return z; -} - object unpacker::data() { - return object(m_ctx->data()); + return m_ctx->data(); } -void unpacker::reset() +void unpacker::reset(zone& z) { if(m_off != 0) { expand_buffer(0); } - if(!m_zone->empty()) { - delete m_zone; - m_zone = NULL; - m_zone = new zone(); - } - m_ctx->reset(); + m_ctx->reset(&z); } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 86cfb6e..2730a08 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -40,7 +40,7 @@ struct unpack_error : public std::runtime_error { class unpacker { public: - unpacker(); + unpacker(zone& z); ~unpacker(); public: @@ -60,13 +60,8 @@ public: /*! 5.1. if execute() returns true, take out the parsed object */ object data(); - /*! 5.2. the parsed object is valid until the zone is deleted */ - // Note that once release_zone() from unpacker, you must delete it - // otherwise the memrory will leak. - zone* release_zone(); - - /*! 5.3. after release_zone(), re-initialize unpacker */ - void reset(); + /*! 5.2. re-initialize unpacker with next zone */ + void reset(zone& z); public: // These functions are usable when non-MessagePack message follows after @@ -85,8 +80,6 @@ public: void remove_nonparsed_buffer(); private: - zone* m_zone; - struct context; context* m_ctx; @@ -94,9 +87,12 @@ private: size_t m_used; size_t m_free; size_t m_off; + +private: void expand_buffer(size_t len); private: + unpacker(); unpacker(const unpacker&); public: diff --git a/cpp/zone.cpp b/cpp/zone.cpp index 4cc50d7..490bc25 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -20,52 +20,35 @@ namespace msgpack { -// FIXME custom allocator? +zone::zone() { } -void zone::expand_chunk() -{ - cell_t* chunk = (cell_t*)malloc(sizeof(cell_t)*ZONE_CHUNK_SIZE); - if(!chunk) { throw std::bad_alloc(); } - try { - m_pool.push_back(chunk); - } catch (...) { - free(chunk); - throw; - } -} +zone::~zone() { clear(); } void zone::clear() { - if(!m_pool.empty()) { - size_t base_size = m_used / ZONE_CHUNK_SIZE; - size_t extend_size = m_used % ZONE_CHUNK_SIZE; - for(size_t b=0; b < base_size; ++b) { - cell_t* c(m_pool[b]); - for(size_t e=0; e < ZONE_CHUNK_SIZE; ++e) { - reinterpret_cast(c[e].data)->~object_class(); + for(std::vector::iterator it(m_ptrs.begin()), it_end(m_ptrs.end()); + it != it_end; ++it) { + free(*it); + } + m_ptrs.clear(); +} + +char* zone::realloc(char* ptr, size_t count) +{ + if(ptr == NULL) { + return zone::malloc(count); + } else { + for(std::vector::reverse_iterator it(m_ptrs.rbegin()), it_end(m_ptrs.rend()); + it != it_end; ++it) { + if(*it == ptr) { + char* tmp = (char*)::realloc(ptr, count); + if(!tmp) { throw std::bad_alloc(); } + *it = tmp; + return tmp; } } - cell_t* c(m_pool.back()); - for(size_t e=0; e < extend_size; ++e) { - reinterpret_cast(c[e].data)->~object_class(); - } - - for(pool_t::iterator it(m_pool.begin()), it_end(m_pool.end()); - it != it_end; - ++it) { - free(*it); - } - m_pool.clear(); + throw std::bad_alloc(); } - m_used = 0; - - for(user_finalizer_t::reverse_iterator it(m_user_finalizer.rbegin()), - it_end(m_user_finalizer.rend()); - it != it_end; - ++it) { - it->call(); - } - m_user_finalizer.clear(); } diff --git a/cpp/zone.hpp b/cpp/zone.hpp new file mode 100644 index 0000000..f8c9cba --- /dev/null +++ b/cpp/zone.hpp @@ -0,0 +1,70 @@ +// +// MessagePack for C++ memory pool +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_ZONE_HPP__ +#define MSGPACK_ZONE_HPP__ + +#include "msgpack/object.hpp" +#include +#include + +namespace msgpack { + + +class zone { +public: + zone(); + ~zone(); + +public: + char* malloc(size_t count); + char* realloc(char* ptr, size_t count); + object* malloc_container(size_t count); + + void clear(); + +private: + std::vector m_ptrs; + +private: + zone(const zone&); +}; + + +inline char* zone::malloc(size_t count) +{ + char* ptr = (char*)::malloc(count); + if(!ptr) { throw std::bad_alloc(); } + try { + m_ptrs.push_back(ptr); + } catch (...) { + free(ptr); + throw; + } + return ptr; +} + +inline object* zone::malloc_container(size_t count) +{ + return (object*)zone::malloc(sizeof(object)*count); +} + + +} // namespace msgpack + +#endif /* msgpack/zone.hpp */ + diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb deleted file mode 100644 index 8af8da2..0000000 --- a/cpp/zone.hpp.erb +++ /dev/null @@ -1,202 +0,0 @@ -// -// MessagePack for C++ memory pool -// -// Copyright (C) 2008 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#ifndef MSGPACK_ZONE_HPP__ -#define MSGPACK_ZONE_HPP__ -#include - -#include "msgpack/object.hpp" -#include -#include -#include - -#ifndef MSGPACK_ZONE_CHUNK_SIZE -#define MSGPACK_ZONE_CHUNK_SIZE 1024 -#endif - -namespace msgpack { - - -static const size_t ZONE_CHUNK_SIZE = MSGPACK_ZONE_CHUNK_SIZE; - - -class zone { -public: - zone() : m_used(0) { } - ~zone() { clear(); } - -public: - template - void push_finalizer(void (*func)(void* obj, void* user), T* obj, void* user); - -public: - object_nil* nnil () { return new (alloc()) object_nil(); } - object_true* ntrue () { return new (alloc()) object_true(); } - object_false* nfalse () { return new (alloc()) object_false(); } - object_u8* nu8 (uint8_t v) { return new (alloc()) object_u8(v); } - object_u16* nu16 (uint16_t v) { return new (alloc()) object_u16(v); } - object_u32* nu32 (uint32_t v) { return new (alloc()) object_u32(v); } - object_u64* nu64 (uint64_t v) { return new (alloc()) object_u64(v); } - object_i8* ni8 (int8_t v) { return new (alloc()) object_i8(v); } - object_i16* ni16 (int16_t v) { return new (alloc()) object_i16(v); } - object_i32* ni32 (int32_t v) { return new (alloc()) object_i32(v); } - object_i64* ni64 (int64_t v) { return new (alloc()) object_i64(v); } - object_float* nfloat (float v) { return new (alloc()) object_float(v); } - object_double* ndouble(double v) { return new (alloc()) object_double(v); } - - - object_mutable_raw_ref* nraw_ref(char* ptr, uint32_t len) - { return new (alloc()) object_mutable_raw_ref(ptr, len); } - - object_raw_ref* nraw_ref(const char* ptr, uint32_t len) - { return new (alloc()) object_raw_ref(ptr, len); } - - object_mutable_raw_ref* nraw_copy(const char* ptr, uint32_t len) - { - char* copy = (char*)malloc(len); - if(!copy) { throw std::bad_alloc(); } - object_mutable_raw_ref* o; - try { - o = new (alloc()) object_mutable_raw_ref(copy, len); - push_finalizer(&zone::finalize_free, NULL, copy); - } catch (...) { - free(copy); - throw; - } - memcpy(copy, ptr, len); - return o; - } - - - object_mutable_raw_ref* nraw_cstr_ref(char* str) - { return nraw_ref(str, strlen(str)); } - - object_raw_ref* nraw_cstr_ref(const char* str) - { return nraw_ref(str, strlen(str)); } - - object_mutable_raw_ref* nraw_cstr_copy(const char* str) - { return nraw_copy(str, strlen(str)); } - - - object_array* narray() - { return new (alloc()) object_array(); } - - object_array* narray(size_t reserve_size) - { return new (alloc()) object_array(reserve_size); } - - object_map* nmap() - { return new (alloc()) object_map(); } - -<% GENERATION_SIZE = 16 %> -<% 1.upto(GENERATION_SIZE) {|i| %> - object_array* narray(<% 1.upto(i-1) {|n| %>object o<%=n%>, <% } %>object o<%=i%>) - { object_array* a = new (alloc()) object_array(<%=i%>); - <% 1.upto(i) {|n| %>a->push_back(o<%=n%>); - <% } %>return a; } -<% } %> - -<% 1.upto(GENERATION_SIZE) {|i| %> - object_map* nmap(<% 1.upto(i-1) {|n| %>object k<%=n%>, object v<%=n%>, <% } %>object k<%=i%>, object v<%=i%>) - { object_map* m = new (alloc()) object_map(); - <% 1.upto(i) {|n| %>m->store(k<%=n%>, v<%=n%>); - <% } %>return m; } -<% } %> - -public: - void clear(); - bool empty() const; - -private: - void* alloc(); - -private: - size_t m_used; - - static const size_t MAX_OBJECT_SIZE = - sizeof(object_raw_ref) > sizeof(object_array) - ? ( sizeof(object_raw_ref) > sizeof(object_map) - ? sizeof(object_raw_ref) - : sizeof(object_map) - ) - : ( sizeof(object_array) > sizeof(object_map) - ? sizeof(object_array) - : sizeof(object_map) - ) - ; - - struct cell_t { - char data[MAX_OBJECT_SIZE]; - }; - - typedef std::vector pool_t; - pool_t m_pool; - - - class finalizer { - public: - finalizer(void (*func)(void*, void*), void* obj, void* user) : - m_obj(obj), m_user(user), m_func(func) {} - void call() { (*m_func)(m_obj, m_user); } - private: - void* m_obj; - void* m_user; - void (*m_func)(void*, void*); - }; - - typedef std::vector user_finalizer_t; - user_finalizer_t m_user_finalizer; - -private: - void expand_chunk(); - -public: - static void finalize_free(void* obj, void* user) - { free(user); } - -private: - zone(const zone&); -}; - - -template -inline void zone::push_finalizer(void (*func)(void* obj, void* user), T* obj, void* user) -{ - m_user_finalizer.push_back( finalizer( - func, reinterpret_cast(obj), - user) ); -} - -inline bool zone::empty() const -{ - return m_used == 0 && m_user_finalizer.empty(); -} - -inline void* zone::alloc() -{ - if(m_pool.size() <= m_used/ZONE_CHUNK_SIZE) { - expand_chunk(); - } - void* data = m_pool[m_used/ZONE_CHUNK_SIZE][m_used%ZONE_CHUNK_SIZE].data; - ++m_used; - return data; -} - - -} // namespace msgpack - -#endif /* msgpack/zone.hpp */ - diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 22d9911..928b581 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -146,7 +146,6 @@ msgpack_pack_inline_func(uint32)(msgpack_pack_user x, uint32_t d) msgpack_pack_inline_func(uint64)(msgpack_pack_user x, uint64_t d) { - // FIXME optimization const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -177,7 +176,6 @@ msgpack_pack_inline_func(int32)(msgpack_pack_user x, int32_t d) msgpack_pack_inline_func(int64)(msgpack_pack_user x, int64_t d) { - // FIXME optimization const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 8197fa4..a6d7dcb 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -279,7 +279,7 @@ _push: c = &stack[top-1]; switch(c->ct) { case CT_ARRAY_ITEM: - msgpack_unpack_callback(array_item)(user, c->obj, obj); + msgpack_unpack_callback(array_item)(user, &c->obj, obj); if(--c->count == 0) { obj = c->obj; --top; @@ -292,7 +292,7 @@ _push: c->ct = CT_MAP_VALUE; goto _header_again; case CT_MAP_VALUE: - msgpack_unpack_callback(map_item)(user, c->obj, c->map_key, obj); + msgpack_unpack_callback(map_item)(user, &c->obj, c->map_key, obj); if(--c->count == 0) { obj = c->obj; --top; diff --git a/ruby/unpack.c b/ruby/unpack.c index 8439c02..b948aa6 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -94,14 +94,14 @@ static inline VALUE template_callback_false(msgpack_unpack_context* x) static inline VALUE template_callback_array(msgpack_unpack_context* x, unsigned int n) { return rb_ary_new2(n); } -static inline void template_callback_array_item(msgpack_unpack_context* x, VALUE c, VALUE o) -{ rb_ary_push(c, o); } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] +static inline void template_callback_array_item(msgpack_unpack_context* x, VALUE* c, VALUE o) +{ rb_ary_push(*c, o); } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] static inline VALUE template_callback_map(msgpack_unpack_context* x, unsigned int n) { return rb_hash_new(); } -static inline void template_callback_map_item(msgpack_unpack_context* x, VALUE c, VALUE k, VALUE v) -{ rb_hash_aset(c, k, v); } +static inline void template_callback_map_item(msgpack_unpack_context* x, VALUE* c, VALUE k, VALUE v) +{ rb_hash_aset(*c, k, v); } static inline VALUE template_callback_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l) { return rb_str_new(p, l); } From b790df530aa28099aa130dafad0a591004b88650 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0021/1172] lang/c/msgpack: C++ binding: safer memory managent git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@68 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/Makefile.am | 2 ++ cpp/test.cpp | 16 ++++++------- cpp/unpack.cpp | 48 +++++++++++++++++++++++++++++++-------- cpp/unpack.hpp | 60 +++++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 102 insertions(+), 24 deletions(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 8b13bea..e244763 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -14,9 +14,11 @@ nobase_include_HEADERS = \ msgpack/type.hpp \ msgpack/type/array.hpp \ msgpack/type/boolean.hpp \ + msgpack/type/float.hpp \ msgpack/type/integer.hpp \ msgpack/type/map.hpp \ msgpack/type/nil.hpp \ + msgpack/type/raw.hpp \ msgpack/type/tuple.hpp noinst_HEADERS = \ diff --git a/cpp/test.cpp b/cpp/test.cpp index dff9101..c632ec5 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -152,9 +152,7 @@ int main(void) unsigned num_msg = 0; static const size_t RESERVE_SIZE = 32;//*1024; - std::auto_ptr pz(new zone()); - - unpacker pac(*pz); + unpacker pac; while(stream.good() && total_bytes > 0) { @@ -179,13 +177,15 @@ int main(void) // 5.1. take out the parsed object object o = pac.data(); - // do something using pz and o + // 5.2 release the zone + std::auto_ptr olife( pac.release_zone() ); + + // 5.3 re-initialize the unpacker */ + pac.reset(); + + // do some with the o and olife std::cout << "message parsed: " << o << std::endl; ++num_msg; - - // 5.3 re-initialize unpacker with next zone */ - pz.reset(new zone()); - pac.reset(*pz); } } diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 71593a0..ce39afd 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -178,12 +178,13 @@ private: }; -unpacker::unpacker(zone& z) : - m_ctx(new context(&z)), +unpacker::unpacker() : m_buffer(NULL), m_used(0), m_free(0), - m_off(0) + m_off(0), + m_zone(new zone()), + m_ctx(new context(&*m_zone)) { } @@ -201,14 +202,14 @@ void unpacker::expand_buffer(size_t len) else { next_size = UNPACKER_INITIAL_BUFFER_SIZE; } while(next_size < len + m_used) { next_size *= 2; } - m_buffer = m_ctx->user()->realloc(m_buffer, next_size); + m_buffer = m_zone->realloc(m_buffer, next_size); m_free = next_size - m_used; } else { size_t next_size = UNPACKER_INITIAL_BUFFER_SIZE; while(next_size < len + m_used - m_off) { next_size *= 2; } - char* tmp = m_ctx->user()->malloc(next_size); + char* tmp = m_zone->malloc(next_size); memcpy(tmp, m_buffer+m_off, m_used-m_off); m_buffer = tmp; @@ -226,20 +227,49 @@ bool unpacker::execute() } else if(ret == 0) { return false; } else { - expand_buffer(0); return true; } } +zone* unpacker::release_zone() +{ + zone* n = new zone(); + std::auto_ptr old(m_zone.release()); + m_zone.reset(n); + + //std::auto_ptr old(new zone()); + //m_zone.swap(old); + + // move all bytes in m_buffer to new buffer from the new zone + if(m_used <= m_off) { + m_buffer = NULL; + m_used = 0; + m_free = 0; + m_off = 0; + } else { + try { + expand_buffer(0); + } catch (...) { + // m_zone.swap(old); + zone* tmp = old.release(); + old.reset(m_zone.release()); + m_zone.reset(tmp); + throw; + } + } + m_ctx->user(&*m_zone); + return old.release(); +} + object unpacker::data() { return m_ctx->data(); } -void unpacker::reset(zone& z) +void unpacker::reset() { - if(m_off != 0) { expand_buffer(0); } - m_ctx->reset(&z); + if(m_off != 0) { std::auto_ptr old(release_zone()); } + m_ctx->reset(); } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 2730a08..07c52e7 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -20,6 +20,7 @@ #include "msgpack/object.hpp" #include "msgpack/zone.hpp" +#include #include #ifndef MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE @@ -40,7 +41,7 @@ struct unpack_error : public std::runtime_error { class unpacker { public: - unpacker(zone& z); + unpacker(); ~unpacker(); public: @@ -60,8 +61,52 @@ public: /*! 5.1. if execute() returns true, take out the parsed object */ object data(); - /*! 5.2. re-initialize unpacker with next zone */ - void reset(zone& z); + /*! 5.2. the object is valid until the zone is deleted */ + // Note that once release_zone() from unpacker, you must delete it + // otherwise the memrory will leak. + zone* release_zone(); + + /*! 5.3. after release_zone(), re-initialize unpacker */ + void reset(); + + + // Basic usage of the unpacker is as following: + // + // msgpack::unpacker pac; + // + // while( /* readable */ ) { + // + // // 1. + // pac.reserve(1024); + // + // // 2. + // ssize_t bytes = + // read(the_source, pac.buffer, pac.buffer_capacity()); + // + // // error handling ... + // + // // 3. + // pac.buffer_consumed(bytes); + // + // // 4. + // while(pac.execute()) { + // // 5.1 + // object o = pac.data(); + // + // // 5.2 + // std::auto_ptr olife( pac.release_zone() ); + // + // // boost::shared_ptr is also usable: + // // boost::shared_ptr olife( pac.release_zone() ); + // + // // 5.3 + // pac.reset(); + // + // // do some with the object with the old zone. + // do_something(o, olife); + // } + // } + // public: // These functions are usable when non-MessagePack message follows after @@ -80,19 +125,20 @@ public: void remove_nonparsed_buffer(); private: - struct context; - context* m_ctx; - char* m_buffer; size_t m_used; size_t m_free; size_t m_off; + std::auto_ptr m_zone; + + struct context; + context* m_ctx; + private: void expand_buffer(size_t len); private: - unpacker(); unpacker(const unpacker&); public: From a7936ba05b15ec7a19e8dc75667fec2df13b7ea7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0022/1172] lang/c/msgpack: C++ binding: implemented msgpack::object >> packer git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@69 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.cpp | 11 ++---- cpp/object.hpp | 92 ++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 92 insertions(+), 11 deletions(-) diff --git a/cpp/object.cpp b/cpp/object.cpp index 276732c..45cfb5f 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -81,7 +81,7 @@ std::ostream& operator<< (std::ostream& s, const object o) default: // FIXME - s << "#" << std::endl; + s << "#"; } return s; } @@ -113,7 +113,7 @@ bool operator==(const object x, const object y) for(object* px(x.via.container.ptr), * const pxend(x.via.container.ptr + x.via.container.size), * py(y.via.container.ptr); - px != pxend; ++px, ++py) { + px < pxend; ++px, ++py) { if(*px != *py) { return false; } } return true; @@ -124,7 +124,7 @@ bool operator==(const object x, const object y) for(object* px(x.via.container.ptr), * const pxend(x.via.container.ptr + x.via.container.size*2), * py(y.via.container.ptr); - px != pxend; ++px, ++py) { + px < pxend; ++px, ++py) { if(*px != *py) { return false; } } return true; @@ -135,10 +135,5 @@ bool operator==(const object x, const object y) } -// FIXME -//template -//const object& operator>> (const object& v, packer& o); - - } // namespace msgpack diff --git a/cpp/object.hpp b/cpp/object.hpp index 7007b90..a5e8682 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include namespace msgpack { @@ -46,7 +47,9 @@ struct object { uint32_t size; } ref; } via; - // FIXME template operator T() { T v; convert(*this, v); return v; }; + + template + operator T() { T v; convert(v, *this); return v; }; }; std::ostream& operator<< (std::ostream& s, const object o); @@ -55,6 +58,9 @@ bool operator==(const object x, const object y); inline bool operator!=(const object x, const object y) { return !(x == y); } +inline object& operator<< (object& v, object o) + { v = o; return v; } + template const object& operator>> (const object& v, packer& o); @@ -73,7 +79,7 @@ namespace type { template inline T& operator<< (T& v, object o) { - v = o; + v.msgpack_unpack(o); return v; } @@ -87,7 +93,7 @@ namespace type { template inline const T& operator>> (const T& v, packer& o) { - detail::pack_copy(v.pack(), o); + detail::pack_copy(v.msgpack_pack(), o); return v; } @@ -118,6 +124,86 @@ inline void pack(T& v, Stream& s) } + +template +const object& operator>> (const object& v, packer& o) +{ + switch(v.type) { + case type::NIL: + o.pack_nil(); + return v; + + case type::BOOLEAN: + if(v.via.boolean) { + o.pack_true(); + } else { + o.pack_false(); + } + return v; + + case type::POSITIVE_INTEGER: + if(v.via.u64 <= (uint64_t)std::numeric_limits::max()) { + if(v.via.u64 <= (uint16_t)std::numeric_limits::max()) { + o.pack_uint8(v.via.u64); + } else { + o.pack_uint16(v.via.u64); + } + } else { + if(v.via.u64 <= (uint64_t)std::numeric_limits::max()) { + o.pack_uint32(v.via.u64); + } else { + o.pack_uint64(v.via.u64); + } + } + return v; + + case type::NEGATIVE_INTEGER: + if(v.via.i64 >= (int64_t)std::numeric_limits::min()) { + if(v.via.i64 >= (int64_t)std::numeric_limits::min()) { + o.pack_int8(v.via.i64); + } else { + o.pack_int16(v.via.i64); + } + } else { + if(v.via.i64 >= (int64_t)std::numeric_limits::min()) { + o.pack_int64(v.via.i64); + } else { + o.pack_int64(v.via.i64); + } + } + return v; + + case type::RAW: + o.pack_raw(v.via.ref.ptr, v.via.ref.size); + return v; + + case type::ARRAY: + o.pack_array(v.via.container.size); + for(object* p(v.via.container.ptr), + * const pend(v.via.container.ptr + v.via.container.size); + p < pend; ++p) { + *p >> o; + } + return v; + // FIXME loop optimiziation + + case type::MAP: + o.pack_map(v.via.container.size); + for(object* p(v.via.container.ptr), + * const pend(v.via.container.ptr + v.via.container.size*2); + p < pend; ) { + *p >> o; ++p; + *p >> o; ++p; + } + return v; + // FIXME loop optimiziation + + default: + throw type_error(); + } +} + + } // namespace msgpack #include "msgpack/type.hpp" From 31659738110f7d73d8fd4fe2d6fc62991ac7912f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0023/1172] lang/c/msgpack: fix compile optimization flag git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@70 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/configure.in | 7 ++++++- cpp/configure.in | 7 ++++++- cpp/test.mk | 9 +++++++++ cpp/zone.cpp | 26 +++++++++--------------- cpp/zone.hpp | 16 +++++++++++++++ msgpack/pack_template.h | 44 ++++++++++++++++++++++++----------------- ruby/extconf.rb | 2 +- ruby/test_pack.rb | 2 ++ ruby/unpack.c | 10 +++++----- 9 files changed, 80 insertions(+), 43 deletions(-) create mode 100644 cpp/test.mk diff --git a/c/configure.in b/c/configure.in index 1cacbc4..0017917 100644 --- a/c/configure.in +++ b/c/configure.in @@ -2,10 +2,15 @@ AC_INIT(pack.c) AM_INIT_AUTOMAKE(msgpackc, 0.1.0) AC_CONFIG_HEADER(config.h) +AC_SUBST(CFLAGS) +if test "" = "$CFLAGS"; then + CFLAGS="-g -O4" +fi + AC_PROG_CC AC_PROG_LIBTOOL -CFLAGS="-O4 $CFLAGS -Wall -I.." +CFLAGS="-O4 -Wall $CFLAGS -I.." AC_OUTPUT([Makefile]) diff --git a/cpp/configure.in b/cpp/configure.in index 5126be4..a60a489 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -2,13 +2,18 @@ AC_INIT(object.cpp) AM_INIT_AUTOMAKE(msgpack, 0.1.0) AC_CONFIG_HEADER(config.h) +AC_SUBST(CXXFLAGS) +if test "" = "$CXXFLAGS"; then + CXXFLAGS="-g -O4" +fi + AC_PROG_CXX AC_PROG_LIBTOOL AC_CHECK_PROG(ERB, erb, erb, [$PATH]) AC_CHECK_LIB(stdc++, main) -CXXFLAGS="-O4 $CXXFLAGS -Wall -I.." +CXXFLAGS="-O4 -Wall $CXXFLAGS -I.." AC_OUTPUT([Makefile]) diff --git a/cpp/test.mk b/cpp/test.mk new file mode 100644 index 0000000..f1beac5 --- /dev/null +++ b/cpp/test.mk @@ -0,0 +1,9 @@ + +CXXFLAGS += -Wall -g -I. -I.. -O4 +LDFLAGS += + +all: test + +test: test.o unpack.o zone.o object.o pack.hpp unpack.hpp zone.hpp object.hpp + $(CXX) test.o unpack.o zone.o object.o $(CXXFLAGS) $(LDFLAGS) -o $@ + diff --git a/cpp/zone.cpp b/cpp/zone.cpp index 490bc25..527cc9c 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -20,10 +20,6 @@ namespace msgpack { -zone::zone() { } - -zone::~zone() { clear(); } - void zone::clear() { for(std::vector::iterator it(m_ptrs.begin()), it_end(m_ptrs.end()); @@ -33,22 +29,18 @@ void zone::clear() m_ptrs.clear(); } -char* zone::realloc(char* ptr, size_t count) +char* zone::realloc_real(char* ptr, size_t count) { - if(ptr == NULL) { - return zone::malloc(count); - } else { - for(std::vector::reverse_iterator it(m_ptrs.rbegin()), it_end(m_ptrs.rend()); - it != it_end; ++it) { - if(*it == ptr) { - char* tmp = (char*)::realloc(ptr, count); - if(!tmp) { throw std::bad_alloc(); } - *it = tmp; - return tmp; - } + for(std::vector::reverse_iterator it(m_ptrs.rbegin()), it_end(m_ptrs.rend()); + it != it_end; ++it) { + if(*it == ptr) { + char* tmp = (char*)::realloc(ptr, count); + if(!tmp) { throw std::bad_alloc(); } + *it = tmp; + return tmp; } - throw std::bad_alloc(); } + throw std::bad_alloc(); } diff --git a/cpp/zone.hpp b/cpp/zone.hpp index f8c9cba..e7e73e1 100644 --- a/cpp/zone.hpp +++ b/cpp/zone.hpp @@ -40,11 +40,18 @@ public: private: std::vector m_ptrs; +private: + char* realloc_real(char* ptr, size_t count); + private: zone(const zone&); }; +inline zone::zone() { } + +inline zone::~zone() { clear(); } + inline char* zone::malloc(size_t count) { char* ptr = (char*)::malloc(count); @@ -58,6 +65,15 @@ inline char* zone::malloc(size_t count) return ptr; } +inline char* zone::realloc(char* ptr, size_t count) +{ + if(ptr == NULL) { + return zone::malloc(count); + } else { + return realloc_real(ptr, count); + } +} + inline object* zone::malloc_container(size_t count) { return (object*)zone::malloc(sizeof(object)*count); diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 928b581..69c7345 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -72,17 +72,21 @@ msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) { if(d < -32) { - if(d < -32768) { // signed 32 + if(d < -32768) { + // signed 32 const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); - } else if(d < -128) { // signed 16 + } else if(d < -128) { + // signed 16 const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); - } else { // signed 8 + } else { + // signed 8 const unsigned char buf[2] = {0xd0, (uint8_t)d}; msgpack_pack_append_buffer(x, buf, 2); } - } else if(d < 128) { // fixnum + } else if(d < 128) { + // fixnum msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); } else { if(d < 256) { @@ -104,21 +108,25 @@ msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) // wrapper msgpack_pack_inline_func(unsigned_int)(msgpack_pack_user x, unsigned int d) { - if(d < 128) { - // fixnum - msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); - } else if(d < 256) { - // unsigned 8 - const unsigned char buf[2] = {0xcc, (uint8_t)d}; - msgpack_pack_append_buffer(x, buf, 2); - } else if(d < 65536) { - // unsigned 16 - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; - msgpack_pack_append_buffer(x, buf, 3); + if(d < 256) { + if(d < 128) { + // fixnum + msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); + } else { + // unsigned 8 + const unsigned char buf[2] = {0xcc, (uint8_t)d}; + msgpack_pack_append_buffer(x, buf, 2); + } } else { - // unsigned 32 - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; - msgpack_pack_append_buffer(x, buf, 5); + if(d < 65536) { + // unsigned 16 + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + // unsigned 32 + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } } } diff --git a/ruby/extconf.rb b/ruby/extconf.rb index 88abb55..e6d4bd6 100644 --- a/ruby/extconf.rb +++ b/ruby/extconf.rb @@ -1,4 +1,4 @@ require 'mkmf' -$CFLAGS << " -I.. -Wall -O9" +$CFLAGS << " -I.. -Wall -O4" create_makefile('msgpack') diff --git a/ruby/test_pack.rb b/ruby/test_pack.rb index 16a8ccf..6873d57 100644 --- a/ruby/test_pack.rb +++ b/ruby/test_pack.rb @@ -21,6 +21,8 @@ check 256 check 65535 check 65536 check -1 +check -32 +check -33 check -128 check -129 check -32768 diff --git a/ruby/unpack.c b/ruby/unpack.c index b948aa6..8ab425c 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -176,13 +176,13 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE args) int ret; if(from >= dlen) { - rb_raise(eUnpackError, "Requested start is after data buffer end."); + rb_raise(eUnpackError, "offset is bigger than data buffer size."); } ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); if(ret < 0) { - rb_raise(eUnpackError, "Parse error."); + rb_raise(eUnpackError, "parse error."); } else if(ret > 0) { mp->user.finished = 1; return ULONG2NUM(from); @@ -242,12 +242,12 @@ static VALUE MessagePack_unpack_impl(VALUE args) ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); if(ret < 0) { - rb_raise(eUnpackError, "Parse error."); + rb_raise(eUnpackError, "parse error."); } else if(ret == 0) { - rb_raise(eUnpackError, "Insufficient bytes."); + rb_raise(eUnpackError, "insufficient bytes."); } else { if(from < dlen) { - rb_raise(eUnpackError, "Extra bytes."); + rb_raise(eUnpackError, "extra bytes."); } return msgpack_unpacker_data(mp); } From f4a6d7faa18eba7fad3e72c8238b0c7463c4294f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0024/1172] lang/c/msgpack: Ruby binding: fix gem warning git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@71 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- ruby/gem/README.txt | 23 +++++++++++++++++++++++ ruby/gengem.sh | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 ruby/gem/README.txt diff --git a/ruby/gem/README.txt b/ruby/gem/README.txt new file mode 100644 index 0000000..7c24b8f --- /dev/null +++ b/ruby/gem/README.txt @@ -0,0 +1,23 @@ +=MessagePack + +== DESCRIPTION: + +Binary-based efficient data interchange format. + + +== LICENSE: + +Copyright (C) 2008 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/ruby/gengem.sh b/ruby/gengem.sh index c8abcc2..000c708 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -6,7 +6,7 @@ cp pack.h gem/ext/ cp rbinit.c gem/ext/ cp unpack.c gem/ext/ cp unpack.h gem/ext/ -cp ../README gem/README.txt +#cp ../README gem/README.txt cp ../msgpack/pack_template.h gem/msgpack/ cp ../msgpack/unpack_define.h gem/msgpack/ cp ../msgpack/unpack_template.h gem/msgpack/ From 2c7f0b2b1aa78b28916c40171dc8fee07d414d64 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0025/1172] lang/c/msgpack: C++ binding: reexamined global operators git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@72 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.hpp | 78 +++++++++++--------- cpp/type/array.hpp | 10 +-- cpp/type/boolean.hpp | 6 +- cpp/type/float.hpp | 18 ++--- cpp/type/integer.hpp | 162 ++++++++++++++++++++--------------------- cpp/type/map.hpp | 32 ++++---- cpp/type/nil.hpp | 14 ++-- cpp/type/raw.hpp | 24 +++--- cpp/type/tuple.hpp.erb | 99 ++++++++++++++----------- cpp/unpack.hpp | 7 ++ 10 files changed, 241 insertions(+), 209 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index a5e8682..8cce14c 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -31,6 +31,18 @@ namespace msgpack { class type_error : public std::bad_cast { }; +namespace type { + static const unsigned char NIL = 0x01; + static const unsigned char BOOLEAN = 0x02; + static const unsigned char POSITIVE_INTEGER = 0x03; + static const unsigned char NEGATIVE_INTEGER = 0x04; + static const unsigned char DOUBLE = 0x05; + static const unsigned char RAW = 0x06; + static const unsigned char ARRAY = 0x07; + static const unsigned char MAP = 0x08; +} + + struct object { unsigned char type; union { @@ -50,6 +62,11 @@ struct object { template operator T() { T v; convert(v, *this); return v; }; + + template + T as() { T v; convert(v, *this); return v; } + + bool is_nil() { return type == type::NIL; } }; std::ostream& operator<< (std::ostream& s, const object o); @@ -58,53 +75,36 @@ bool operator==(const object x, const object y); inline bool operator!=(const object x, const object y) { return !(x == y); } -inline object& operator<< (object& v, object o) +inline object& operator>> (object o, object& v) { v = o; return v; } template -const object& operator>> (const object& v, packer& o); +packer& operator<< (packer& o, const object& v); + namespace type { - static const unsigned char NIL = 0x01; - static const unsigned char BOOLEAN = 0x02; - static const unsigned char POSITIVE_INTEGER = 0x03; - static const unsigned char NEGATIVE_INTEGER = 0x04; - static const unsigned char DOUBLE = 0x05; - static const unsigned char RAW = 0x06; - static const unsigned char ARRAY = 0x07; - static const unsigned char MAP = 0x08; - - template - inline T& operator<< (T& v, object o) + inline T& operator>> (object o, T& v) { v.msgpack_unpack(o); return v; } - - - namespace detail { - template - inline void pack_copy(T v, packer& o) - { pack(v, o); } - } - + template - inline const T& operator>> (const T& v, packer& o) + inline packer& operator<< (packer& o, const T& v) { - detail::pack_copy(v.msgpack_pack(), o); - return v; + pack_copy(v.msgpack_pack(), o); + return o; } - -} // namespace type +} template inline void convert(T& v, object o) { using namespace type; - v << o; + o >> v; } @@ -112,7 +112,7 @@ template inline void pack(T& v, packer& o) { using namespace type; - v >> o; + o << v; } @@ -124,14 +124,20 @@ inline void pack(T& v, Stream& s) } +template +inline void pack_copy(T v, packer& o) +{ + pack(v, o); +} + template -const object& operator>> (const object& v, packer& o) +packer& operator<< (packer& o, const object& v) { switch(v.type) { case type::NIL: o.pack_nil(); - return v; + return o; case type::BOOLEAN: if(v.via.boolean) { @@ -139,7 +145,7 @@ const object& operator>> (const object& v, packer& o) } else { o.pack_false(); } - return v; + return o; case type::POSITIVE_INTEGER: if(v.via.u64 <= (uint64_t)std::numeric_limits::max()) { @@ -155,7 +161,7 @@ const object& operator>> (const object& v, packer& o) o.pack_uint64(v.via.u64); } } - return v; + return o; case type::NEGATIVE_INTEGER: if(v.via.i64 >= (int64_t)std::numeric_limits::min()) { @@ -171,11 +177,11 @@ const object& operator>> (const object& v, packer& o) o.pack_int64(v.via.i64); } } - return v; + return o; case type::RAW: o.pack_raw(v.via.ref.ptr, v.via.ref.size); - return v; + return o; case type::ARRAY: o.pack_array(v.via.container.size); @@ -184,7 +190,7 @@ const object& operator>> (const object& v, packer& o) p < pend; ++p) { *p >> o; } - return v; + return o; // FIXME loop optimiziation case type::MAP: @@ -195,7 +201,7 @@ const object& operator>> (const object& v, packer& o) *p >> o; ++p; *p >> o; ++p; } - return v; + return o; // FIXME loop optimiziation default: diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp index 7066703..703ac55 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/array.hpp @@ -22,13 +22,12 @@ #include namespace msgpack { -namespace type { template -inline std::vector operator<< (std::vector& v, object o) +inline std::vector operator>> (object o, std::vector& v) { - if(o.type != ARRAY) { throw type_error(); } + if(o.type != type::ARRAY) { throw type_error(); } v.resize(o.via.container.size); object* p(o.via.container.ptr); object* const pend(o.via.container.ptr + o.via.container.size); @@ -41,18 +40,17 @@ inline std::vector operator<< (std::vector& v, object o) template -inline const std::vector& operator>> (const std::vector& v, packer& o) +inline packer& operator<< (packer& o, const std::vector& v) { o.pack_array(v.size()); for(typename std::vector::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { pack(*it, o); } - return v; + return o; } -} // namespace type } // namespace msgpack #endif /* msgpack/type/array.hpp */ diff --git a/cpp/type/boolean.hpp b/cpp/type/boolean.hpp index 0538495..e958478 100644 --- a/cpp/type/boolean.hpp +++ b/cpp/type/boolean.hpp @@ -25,7 +25,7 @@ namespace msgpack { namespace type { -inline bool& operator<< (bool& v, object o) +inline bool& operator>> (object o, bool& v) { if(o.type != BOOLEAN) { throw type_error(); } v = o.via.boolean; @@ -34,11 +34,11 @@ inline bool& operator<< (bool& v, object o) template -inline const bool& operator>> (const bool& v, packer o) +inline packer& operator<< (packer& o, const bool& v) { if(v) { o.pack_true(); } else { o.pack_false(); } - return v; + return o; } diff --git a/cpp/type/float.hpp b/cpp/type/float.hpp index 11fa280..2178434 100644 --- a/cpp/type/float.hpp +++ b/cpp/type/float.hpp @@ -22,45 +22,43 @@ #include namespace msgpack { -namespace type { // FIXME check overflow, underflow -inline float& operator<< (float& v, object o) +inline float& operator>> (object o, float& v) { - if(o.type != DOUBLE) { throw type_error(); } + if(o.type != type::DOUBLE) { throw type_error(); } v = o.via.dec; return v; } template -inline const float& operator>> (const float& v, packer o) +inline packer& operator<< (packer& o, const float& v) { o.pack_float(v); - return v; + return o; } -inline double& operator<< (double& v, object o) +inline double& operator>> (object o, double& v) { - if(o.type != DOUBLE) { throw type_error(); } + if(o.type != type::DOUBLE) { throw type_error(); } v = o.via.dec; return v; } template -inline const double& operator>> (const double& v, packer o) +inline packer& operator<< (packer& o, const double& v) { o.pack_double(v); - return v; + return o; } -} // namespace type } // namespace msgpack #endif /* msgpack/type/float.hpp */ diff --git a/cpp/type/integer.hpp b/cpp/type/integer.hpp index 488c9d4..f5841a6 100644 --- a/cpp/type/integer.hpp +++ b/cpp/type/integer.hpp @@ -22,9 +22,9 @@ #include namespace msgpack { + + namespace type { - - namespace detail { template struct convert_integer_sign; @@ -32,11 +32,11 @@ namespace detail { template struct convert_integer_sign { static inline T convert(object o) { - if(o.type == POSITIVE_INTEGER) { + if(o.type == type::POSITIVE_INTEGER) { if(o.via.u64 > (uint64_t)std::numeric_limits::max()) { throw type_error(); } return o.via.u64; - } else if(o.type == NEGATIVE_INTEGER) { + } else if(o.type == type::NEGATIVE_INTEGER) { if(o.via.i64 < (int64_t)-std::numeric_limits::max()) { throw type_error(); } return o.via.i64; @@ -48,7 +48,7 @@ namespace detail { template struct convert_integer_sign { static inline T convert(object o) { - if(o.type == POSITIVE_INTEGER) { + if(o.type == type::POSITIVE_INTEGER) { if(o.via.u64 > (uint64_t)std::numeric_limits::max()) { throw type_error(); } return o.via.u64; @@ -160,83 +160,83 @@ namespace detail { } } // namespace detail - - -inline signed char& operator<< (signed char& v, object o) - { v = detail::convert_integer(o); return v; } - -inline signed short& operator<< (signed short& v, object o) - { v = detail::convert_integer(o); return v; } - -inline signed int& operator<< (signed int& v, object o) - { v = detail::convert_integer(o); return v; } - -inline signed long& operator<< (signed long& v, object o) - { v = detail::convert_integer(o); return v; } - -inline signed long long& operator<< (signed long long& v, object o) - { v = detail::convert_integer(o); return v; } - - -inline unsigned char& operator<< (unsigned char& v, object o) - { v = detail::convert_integer(o); return v; } - -inline unsigned short& operator<< (unsigned short& v, object o) - { v = detail::convert_integer(o); return v; } - -inline unsigned int& operator<< (unsigned int& v, object o) - { v = detail::convert_integer(o); return v; } - -inline unsigned long& operator<< (unsigned long& v, object o) - { v = detail::convert_integer(o); return v; } - -inline unsigned long long& operator<< (unsigned long long& v, object o) - { v = detail::convert_integer(o); return v; } - - -template -inline const signed char& operator>> (const signed char& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const signed short& operator>> (const signed short& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const signed int& operator>> (const signed int& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const signed long& operator>> (const signed long& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const signed long long& operator>> (const signed long long& v, packer o) - { detail::pack_integer(v, o); return v; } - - -template -inline const unsigned char& operator>> (const unsigned char& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const unsigned short& operator>> (const unsigned short& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const unsigned int& operator>> (const unsigned int& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const unsigned long& operator>> (const unsigned long& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const unsigned long long& operator>> (const unsigned long long& v, packer o) - { detail::pack_integer(v, o); return v; } - - } // namespace type + + +inline signed char& operator>> (object o, signed char& v) + { v = type::detail::convert_integer(o); return v; } + +inline signed short& operator>> (object o, signed short& v) + { v = type::detail::convert_integer(o); return v; } + +inline signed int& operator>> (object o, signed int& v) + { v = type::detail::convert_integer(o); return v; } + +inline signed long& operator>> (object o, signed long& v) + { v = type::detail::convert_integer(o); return v; } + +inline signed long long& operator>> (object o, signed long long& v) + { v = type::detail::convert_integer(o); return v; } + + +inline unsigned char& operator>> (object o, unsigned char& v) + { v = type::detail::convert_integer(o); return v; } + +inline unsigned short& operator>> (object o, unsigned short& v) + { v = type::detail::convert_integer(o); return v; } + +inline unsigned int& operator>> (object o, unsigned int& v) + { v = type::detail::convert_integer(o); return v; } + +inline unsigned long& operator>> (object o, unsigned long& v) + { v = type::detail::convert_integer(o); return v; } + +inline unsigned long long& operator>> (object o, unsigned long long& v) + { v = type::detail::convert_integer(o); return v; } + + +template +inline packer& operator<< (packer& o, const signed char& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const signed short& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const signed int& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const signed long& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const signed long long& v) + { type::detail::pack_integer(v, o); return o; } + + +template +inline packer& operator<< (packer& o, const unsigned char& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const unsigned short& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const unsigned int& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const unsigned long& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const unsigned long long& v) + { type::detail::pack_integer(v, o); return o; } + + } // namespace msgpack #endif /* msgpack/type/integer.hpp */ diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index 1efecec..1d5e054 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -23,9 +23,10 @@ #include namespace msgpack { -namespace type { +namespace type { + template class assoc_vector : public std::vector< std::pair > {}; @@ -37,10 +38,13 @@ namespace detail { }; } +} //namespace type + + template -inline assoc_vector& operator<< (assoc_vector& v, object o) +inline type::assoc_vector& operator>> (object o, type::assoc_vector& v) { - if(o.type != MAP) { throw type_error(); } + if(o.type != type::MAP) { throw type_error(); } v.resize(o.via.container.size); object* p(o.via.container.ptr); object* const pend(o.via.container.ptr + o.via.container.size); @@ -49,26 +53,27 @@ inline assoc_vector& operator<< (assoc_vector& v, object o) convert(it->first, *p); ++p; convert(it->second, *p); ++p; } - std::sort(v.begin(), v.end(), detail::pair_first_less()); + std::sort(v.begin(), v.end(), type::detail::pair_first_less()); return v; } template -inline const assoc_vector& operator>> (const assoc_vector& v, packer& o) +inline packer& operator<< (packer& o, const type::assoc_vector& v) { o.pack_map(v.size()); - for(typename assoc_vector::const_iterator it(v.begin()), it_end(v.end()); + for(typename type::assoc_vector::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { pack(it->first, o); pack(it->second, o); } + return o; } template -inline std::map operator<< (std::map& v, object o) +inline std::map operator>> (object o, std::map& v) { - if(o.type != MAP) { throw type_error(); } + if(o.type != type::MAP) { throw type_error(); } object* p(o.via.container.ptr); object* const pend(o.via.container.ptr + o.via.container.size*2); while(p < pend) { @@ -87,7 +92,7 @@ inline std::map operator<< (std::map& v, object o) } template -inline const std::map& operator>> (const std::map& v, packer& o) +inline packer& operator<< (packer& o, const std::map& v) { o.pack_map(v.size()); for(typename std::map::const_iterator it(v.begin()), it_end(v.end()); @@ -95,13 +100,14 @@ inline const std::map& operator>> (const std::map& v, packer& pack(it->first, o); pack(it->second, o); } + return o; } template -inline std::multimap operator<< (std::multimap& v, object o) +inline std::multimap operator>> (object o, std::multimap& v) { - if(o.type != MAP) { throw type_error(); } + if(o.type != type::MAP) { throw type_error(); } object* p(o.via.container.ptr); object* const pend(o.via.container.ptr + o.via.container.size*2); while(p < pend) { @@ -114,7 +120,7 @@ inline std::multimap operator<< (std::multimap& v, object o) } template -inline const std::multimap& operator>> (const std::multimap& v, packer& o) +inline packer& operator<< (packer& o, const std::multimap& v) { o.pack_multimap(v.size()); for(typename std::multimap::const_iterator it(v.begin()), it_end(v.end()); @@ -122,10 +128,10 @@ inline const std::multimap& operator>> (const std::multimap& v, packer pack(it->first, o); pack(it->second, o); } + return o; } -} // namespace type } // namespace msgpack #endif /* msgpack/type/map.hpp */ diff --git a/cpp/type/nil.hpp b/cpp/type/nil.hpp index 73cedb4..ab0c363 100644 --- a/cpp/type/nil.hpp +++ b/cpp/type/nil.hpp @@ -21,26 +21,28 @@ #include "msgpack/object.hpp" namespace msgpack { -namespace type { +namespace type { struct nil { }; -inline nil& operator<< (nil& v, object o) +} // namespace type + + +inline type::nil& operator>> (object o, type::nil& v) { - if(o.type != NIL) { throw type_error(); } + if(o.type != type::NIL) { throw type_error(); } return v; } template -inline const nil& operator>> (const nil& v, packer& o) +inline packer& operator<< (packer& o, const type::nil& v) { o.pack_nil(); - return v; + return o; } -} // namespace type } // namespace msgpack #endif /* msgpack/type/nil.hpp */ diff --git a/cpp/type/raw.hpp b/cpp/type/raw.hpp index 8767238..14e52f4 100644 --- a/cpp/type/raw.hpp +++ b/cpp/type/raw.hpp @@ -23,8 +23,8 @@ #include namespace msgpack { -namespace type { +namespace type { struct raw_ref { raw_ref() : ptr(NULL), size(0) {} @@ -58,41 +58,43 @@ struct raw_ref { } }; -inline raw_ref& operator<< (raw_ref& v, object o) +} // namespace type + + +inline type::raw_ref& operator>> (object o, type::raw_ref& v) { - if(o.type != RAW) { throw type_error(); } + if(o.type != type::RAW) { throw type_error(); } v.ptr = o.via.ref.ptr; v.size = o.via.ref.size; return v; } -inline std::string& operator<< (std::string& v, object o) +inline std::string& operator>> (object o, std::string& v) { - raw_ref r; - r << o; + type::raw_ref r; + o >> r; v.assign(r.ptr, r.size); return v; } template -inline const raw_ref& operator>> (const raw_ref& v, packer& o) +inline packer& operator<< (packer& o, const type::raw_ref& v) { o.pack_raw(v.ptr, v.size); - return v; + return o; } template -inline const std::string& operator>> (const std::string& v, packer& o) +inline packer& operator<< (packer& o, const std::string& v) { o.pack_raw(v.data(), v.size()); - return v; + return o; } -} // namespace type } // namespace msgpack #endif /* msgpack/type/raw.hpp */ diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index 2a8a2dd..c54739a 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -21,15 +21,14 @@ #include "msgpack/object.hpp" namespace msgpack { -namespace type { +namespace type { // FIXME operator== // FIXME operator!= - <% GENERATION_LIMIT = 15 %> -template < typename A0 <%1.upto(GENERATION_LIMIT+1) {|i|%>, typename A<%=i%> = void<%}%> > +template , typename A<%=i%> = void<%}%>> struct tuple; template @@ -60,9 +59,9 @@ private: <%0.upto(GENERATION_LIMIT) {|i|%> <%0.upto(i) {|j|%> -template < typename A0 <%1.upto(i) {|k|%>, typename A<%=k%> <%}%>> -struct tuple_type, A<%=k%> <%}%>>, <%=j%>> : tuple_element> { - tuple_type(tuple, A<%=k%> <%}%>>& x) : tuple_element>(x.a<%=j%>) {} +template , typename A<%=k%><%}%>> +struct tuple_type, A<%=k%><%}%>>, <%=j%>> : tuple_element> { + tuple_type(tuple, A<%=k%> <%}%>>& x) : tuple_element>(x.a<%=j%>) {} }; <%}%> <%}%> @@ -70,46 +69,57 @@ struct tuple_type, A<%=k%> <%}%>>, <%=j%>> : tuple_e <%0.upto(GENERATION_LIMIT) {|i|%> <%0.upto(i) {|j|%> -template < typename A0 <%1.upto(i) {|k|%>, typename A<%=k%> <%}%>> -struct const_tuple_type, A<%=k%> <%}%>>, <%=j%>> : const_tuple_element> { - const_tuple_type(const tuple, A<%=k%> <%}%>>& x) : const_tuple_element>(x.a<%=j%>) {} +template , typename A<%=k%><%}%>> +struct const_tuple_type, A<%=k%><%}%>>, <%=j%>> : const_tuple_element> { + const_tuple_type(const tuple, A<%=k%><%}%>>& x) : const_tuple_element>(x.a<%=j%>) {} }; <%}%> <%}%> <%0.upto(GENERATION_LIMIT) {|i|%> -template < typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> -tuple< A0 <%1.upto(i) {|j|%>, A<%=j%> <%}%>> make_tuple(const A0& a0 <%1.upto(i) {|j|%>, const A<%=j%>& a<%=j%><%}%>) +template , typename A<%=j%><%}%>> +tuple, A<%=j%><%}%>> make_tuple(const A0& a0<%1.upto(i) {|j|%>, const A<%=j%>& a<%=j%><%}%>) { - return tuple< A0 <%1.upto(i) {|j|%>, A<%=j%> <%}%>>(a0 <%1.upto(i) {|j|%>, a<%=j%><%}%>); + return tuple, A<%=j%><%}%>>(a0<%1.upto(i) {|j|%>, a<%=j%><%}%>); } <%}%> +template <> +struct tuple<> { +}; <%0.upto(GENERATION_LIMIT) {|i|%> -template < typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> -struct tuple, A<%=j%> <%}%>> { +template , typename A<%=j%><%}%>> +struct tuple, A<%=j%><%}%>> { tuple() {} - tuple(const A0& _a0 <%1.upto(i) {|j|%>, const A<%=j%>& _a<%=j%><%}%>) : + tuple(const A0& _a0<%1.upto(i) {|j|%>, const A<%=j%>& _a<%=j%><%}%>) : a0(_a0) <%1.upto(i) {|j|%>, a<%=j%>(_a<%=j%>)<%}%> {} tuple(object o) { convert(*this, o); } - template typename tuple_type, A<%=j%> <%}%>>, N>::type& get() - { return tuple_type, A<%=j%> <%}%>>, N>(*this).get(); } - template const typename const_tuple_type, A<%=j%> <%}%>>, N>::type& get() const - { return const_tuple_type, A<%=j%> <%}%>>, N>(*this).get(); } + template typename tuple_type, A<%=j%><%}%>>, N>::type& get() + { return tuple_type, A<%=j%><%}%>>, N>(*this).get(); } + template const typename const_tuple_type, A<%=j%><%}%>>, N>::type& get() const + { return const_tuple_type, A<%=j%><%}%>>, N>(*this).get(); } <%0.upto(i) {|j|%> A<%=j%> a<%=j%>;<%}%> }; <%}%> +} // namespace type + +inline type::tuple<>& operator>> ( + object o, + type::tuple<>& v) { + if(o.type != type::ARRAY) { throw type_error(); } + return v; +} <%0.upto(GENERATION_LIMIT) {|i|%> -template < typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> -tuple, A<%=j%> <%}%>>& operator<< ( - tuple, A<%=j%> <%}%>>& v, - object o) { - if(o.type != ARRAY) { throw type_error(); } +template , typename A<%=j%><%}%>> +type::tuple, A<%=j%><%}%>>& operator>> ( + object o, + type::tuple, A<%=j%><%}%>>& v) { + if(o.type != type::ARRAY) { throw type_error(); } if(o.via.container.size < <%=i+1%>) { throw type_error(); } <%0.upto(i) {|j|%> convert>(v.template get<<%=j%>>(), o.via.container.ptr[<%=j%>]);<%}%> @@ -118,6 +128,26 @@ tuple, A<%=j%> <%}%>>& operator<< ( <%}%> +template +const packer& operator<< ( + packer& o, + const type::tuple<>& v) { + o.pack_array(0); + return o; +} +<%0.upto(GENERATION_LIMIT) {|i|%> +template , typename A<%=j%><%}%>> +const packer& operator<< ( + packer& o, + const type::tuple, A<%=j%><%}%>>& v) { + o.pack_array(<%=i+1%>); + <%0.upto(i) {|j|%> + pack(v.template get<<%=j%>>(), o);<%}%> + return o; +} +<%}%> + + // FIXME /* template @@ -128,7 +158,7 @@ struct tuple_just { A0 a0; static inline void convert(object o, tuple_just& v) { - if(o.type != ARRAY) { throw type_error(); } + if(o.type != type::ARRAY) { throw type_error(); } if(o.v.container.size != 1) { throw type_error(); } msgpack::convert(o.v.container.ptr[0], v.a0); } @@ -140,7 +170,7 @@ struct tuple_just { A1 a1; static inline void convert(object o, tuple_just& v) { - if(o.type != ARRAY) { throw type_error(); } + if(o.type != type::ARRAY) { throw type_error(); } if(o.v.container.size != 2) { throw type_error(); } msgpack::convert(o.v.container.ptr[0], v.a0); msgpack::convert(o.v.container.ptr[1], v.a1); @@ -149,24 +179,7 @@ struct tuple_just { */ - -<%0.upto(GENERATION_LIMIT) {|i|%> -template < typename Stream , typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> -const tuple, A<%=j%> <%}%>>& operator>> ( - const tuple, A<%=j%> <%}%>>& v, - packer o) { - o.pack_array(<%=i+1%>); - <%0.upto(i) {|j|%> - pack(v.template get<<%=j%>>(), o);<%}%> - return v; -} -<%}%> - - - -} // namespace type } // namespace msgpack #endif /* msgpack/type/tuple.hpp */ - diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 07c52e7..de613a4 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -124,6 +124,10 @@ public: // Note that reset() leaves non-parsed buffer. void remove_nonparsed_buffer(); + /*! skip specified size of non-parsed buffer, leaving the buffer */ + // Note the size must be smaller than nonparsed_size() + void skip_nonparsed_buffer(size_t len); + private: char* m_buffer; size_t m_used; @@ -177,6 +181,9 @@ inline size_t unpacker::parsed_size() const inline void unpacker::remove_nonparsed_buffer() { m_used = m_off; } +inline void unpacker::skip_nonparsed_buffer(size_t len) + { m_off += len; } + inline object unpack(const char* data, size_t len, zone& z, size_t* off = NULL) { From 921b0ff62ec99e9339d6cfaa6ba724ae7bb5f9ce Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0026/1172] lang/c/msgpack: C++ binding: pack() git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@73 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.hpp | 41 +++++++++++++++++------------------------ cpp/test.cpp | 4 ++-- cpp/type/array.hpp | 2 +- cpp/type/boolean.hpp | 4 +--- cpp/type/integer.hpp | 18 +++++++++--------- cpp/type/map.hpp | 12 ++++++------ cpp/type/tuple.hpp.erb | 2 +- cpp/unpack.hpp | 2 +- 8 files changed, 38 insertions(+), 47 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 8cce14c..77d55bb 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -83,51 +83,44 @@ packer& operator<< (packer& o, const object& v); -namespace type { - template - inline T& operator>> (object o, T& v) - { - v.msgpack_unpack(o); - return v; - } - - template - inline packer& operator<< (packer& o, const T& v) - { - pack_copy(v.msgpack_pack(), o); - return o; - } +template +inline T& operator>> (object o, T& v) +{ + v.msgpack_unpack(o); + return v; +} + +template +inline packer& operator<< (packer& o, const T& v) +{ + o << v.msgpack_pack(); + return o; } template inline void convert(T& v, object o) { - using namespace type; o >> v; } - template -inline void pack(T& v, packer& o) +inline void pack(packer& o, const T& v) { - using namespace type; o << v; } - template -inline void pack(T& v, Stream& s) +inline void pack(Stream& s, const T& v) { packer pk(s); - pack(v, pk); + pack(pk, v); } - template -inline void pack_copy(T v, packer& o) +inline void pack_copy(packer& o, T v) { - pack(v, o); + pack(o, v); } diff --git a/cpp/test.cpp b/cpp/test.cpp index c632ec5..02d461c 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -26,7 +26,7 @@ public: try { std::stringstream s; - pack(should, s); + pack(s, should); std::string str(s.str()); object ro = unpack(str.data(), str.size(), m_zone); std::cout << ro << std::endl; @@ -139,7 +139,7 @@ int main(void) // send message { for(unsigned i=0; i < TASK_REPEAT; ++i) { - pack(task, stream); + pack(stream, task); } std::cout << "send " << stream.str().size() << " bytes" << std::endl; } diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp index 703ac55..0522d4c 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/array.hpp @@ -45,7 +45,7 @@ inline packer& operator<< (packer& o, const std::vector& v) o.pack_array(v.size()); for(typename std::vector::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(*it, o); + pack(o, *it); } return o; } diff --git a/cpp/type/boolean.hpp b/cpp/type/boolean.hpp index e958478..60f1714 100644 --- a/cpp/type/boolean.hpp +++ b/cpp/type/boolean.hpp @@ -22,12 +22,11 @@ #include namespace msgpack { -namespace type { inline bool& operator>> (object o, bool& v) { - if(o.type != BOOLEAN) { throw type_error(); } + if(o.type != type::BOOLEAN) { throw type_error(); } v = o.via.boolean; return v; } @@ -42,7 +41,6 @@ inline packer& operator<< (packer& o, const bool& v) } -} // namespace type } // namespace msgpack #endif /* msgpack/type/bool.hpp */ diff --git a/cpp/type/integer.hpp b/cpp/type/integer.hpp index f5841a6..3fd57b4 100644 --- a/cpp/type/integer.hpp +++ b/cpp/type/integer.hpp @@ -70,19 +70,19 @@ namespace detail { template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) + static inline void pack(packer& o, T v) { o.pack_int8(v); } }; template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) + static inline void pack(packer& o, T v) { o.pack_uint8(v); } }; template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) { + static inline void pack(packer& o, T v) { if( (int16_t)v <= (int16_t)std::numeric_limits::max() && (int16_t)v >= (int16_t)std::numeric_limits::min()) { o.pack_int8(v); } @@ -92,7 +92,7 @@ namespace detail { template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) { + static inline void pack(packer& o, T v) { if( (uint16_t)v <= (uint16_t)std::numeric_limits::max()) { o.pack_uint8(v); } else { o.pack_uint16(v); } @@ -101,7 +101,7 @@ namespace detail { template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) { + static inline void pack(packer& o, T v) { if( (int32_t)v <= (int32_t)std::numeric_limits::max() && (int32_t)v >= (int32_t)std::numeric_limits::min()) { o.pack_int8(v); } @@ -114,7 +114,7 @@ namespace detail { template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) { + static inline void pack(packer& o, T v) { if( (uint32_t)v <= (uint32_t)std::numeric_limits::max()) { o.pack_uint8(v); } else if( (uint32_t)v <= (uint32_t)std::numeric_limits::max()) @@ -125,7 +125,7 @@ namespace detail { template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) { + static inline void pack(packer& o, T v) { if( (int64_t)v <= (int64_t)std::numeric_limits::max() && (int64_t)v >= (int64_t)std::numeric_limits::min()) { o.pack_int8(v); } @@ -141,7 +141,7 @@ namespace detail { template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) { + static inline void pack(packer& o, T v) { if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) { o.pack_uint8(v); } else if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) @@ -156,7 +156,7 @@ namespace detail { template static inline void pack_integer(T v, packer& o) { - pack_integer_size_sign::is_signed>::pack(v, o); + pack_integer_size_sign::is_signed>::pack(o, v); } } // namespace detail diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index 1d5e054..3b544df 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -63,8 +63,8 @@ inline packer& operator<< (packer& o, const type::assoc_vector::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(it->first, o); - pack(it->second, o); + pack(o, it->first); + pack(o, it->second); } return o; } @@ -97,8 +97,8 @@ inline packer& operator<< (packer& o, const std::map& v) o.pack_map(v.size()); for(typename std::map::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(it->first, o); - pack(it->second, o); + pack(o, it->first); + pack(o, it->second); } return o; } @@ -125,8 +125,8 @@ inline packer& operator<< (packer& o, const std::multimap& o.pack_multimap(v.size()); for(typename std::multimap::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(it->first, o); - pack(it->second, o); + pack(o, it->first); + pack(o, it->second); } return o; } diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index c54739a..397f660 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -142,7 +142,7 @@ const packer& operator<< ( const type::tuple, A<%=j%><%}%>>& v) { o.pack_array(<%=i+1%>); <%0.upto(i) {|j|%> - pack(v.template get<<%=j%>>(), o);<%}%> + pack(o, v.template get<<%=j%>>());<%}%> return o; } <%}%> diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index de613a4..ffee60d 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -81,7 +81,7 @@ public: // // // 2. // ssize_t bytes = - // read(the_source, pac.buffer, pac.buffer_capacity()); + // read(the_source, pac.buffer(), pac.buffer_capacity()); // // // error handling ... // From 1ad04b22d8d3e5267ceec7fc1527651ee0eb0ffe Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0027/1172] lang/c/msgpack: divide pack_raw() into pack_raw() and pack_raw_body() git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@74 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/bench.c | 3 +- c/pack.h | 4 +- cpp/bench.cpp | 3 +- cpp/object.hpp | 3 +- cpp/pack.hpp | 131 ++-------------------------------------- cpp/type/raw.hpp | 6 +- cpp/unpack.cpp | 4 +- msgpack/pack_template.h | 7 ++- ruby/pack.c | 6 +- 9 files changed, 28 insertions(+), 139 deletions(-) diff --git a/c/bench.c b/c/bench.c index fa717c0..d72a10d 100644 --- a/c/bench.c +++ b/c/bench.c @@ -279,7 +279,8 @@ void bench_msgpack(void) unsigned int i; msgpack_pack_array(mpk, TASK_STR_LEN); for(i=0; i < TASK_STR_LEN; ++i) { - msgpack_pack_raw(mpk, TASK_STR_PTR, i); + msgpack_pack_raw(mpk, i); + msgpack_pack_raw_body(mpk, TASK_STR_PTR, i); } } show_timer(mpkbuf.length); diff --git a/c/pack.h b/c/pack.h index c6cadf4..ec0683e 100644 --- a/c/pack.h +++ b/c/pack.h @@ -54,8 +54,8 @@ void msgpack_pack_true(msgpack_pack_t* ctx); void msgpack_pack_false(msgpack_pack_t* ctx); void msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); void msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); -void msgpack_pack_string(msgpack_pack_t* ctx, const char* b); -void msgpack_pack_raw(msgpack_pack_t* ctx, const void* b, size_t l); +void msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); +void msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); #ifdef __cplusplus diff --git a/cpp/bench.cpp b/cpp/bench.cpp index 1aad807..aa303fa 100644 --- a/cpp/bench.cpp +++ b/cpp/bench.cpp @@ -142,7 +142,8 @@ void bench_msgpack_str() msgpack::packer pk(buf); pk.pack_array(TASK_STR_LEN); for(unsigned int i=0; i < TASK_STR_LEN; ++i) { - pk.pack_raw(TASK_STR_PTR, i); + pk.pack_raw(i); + pk.pack_raw_body(TASK_STR_PTR, i); } } timer.show_stat(buf.size()); diff --git a/cpp/object.hpp b/cpp/object.hpp index 77d55bb..13402cc 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -173,7 +173,8 @@ packer& operator<< (packer& o, const object& v) return o; case type::RAW: - o.pack_raw(v.via.ref.ptr, v.via.ref.size); + o.pack_raw(v.via.ref.size); + o.pack_raw_body(v.via.ref.ptr, v.via.ref.size); return o; case type::ARRAY: diff --git a/cpp/pack.hpp b/cpp/pack.hpp index fe470b6..4fda4ff 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -47,7 +47,8 @@ public: void pack_false() { pack_false_impl(m_stream); } void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } - void pack_raw(const char* b, size_t l) { pack_raw_impl(m_stream, (const void*)b, l); } + void pack_raw(size_t l) { pack_raw_impl(m_stream, l); } + void pack_raw_body(const char* b, size_t l) { pack_raw_body_impl(m_stream, b, l); } private: static void pack_int_impl(Stream& x, int d); @@ -67,7 +68,9 @@ private: static void pack_false_impl(Stream& x); static void pack_array_impl(Stream& x, unsigned int n); static void pack_map_impl(Stream& x, unsigned int n); - static void pack_raw_impl(Stream& x, const void* b, size_t l); + static void pack_raw_impl(Stream& x, size_t l); + static void pack_raw_body_impl(Stream& x, const void* b, size_t l); + static void append_buffer(Stream& x, const unsigned char* buf, unsigned int len) { x.write((const char*)buf, len); } @@ -90,130 +93,6 @@ template packer::packer(Stream& s) : m_stream(s) { } -/* -class dynamic_stream { -public: - template - dynamic_stream(Stream& s); -public: - void write(const char* buf, size_t len) - { (*m_function)(m_object, buf, len); } -private: - void* m_object; - void (*m_function)(void* object, const char* buf, size_t len); -private: - struct write_trampoline; -}; - - -class dynamic_packer { -public: - template - dynamic_packer(Stream& s); - -public: - void pack_int(int d) { pack_int_impl(m_stream, d); } - void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } - void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } - void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } - void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } - void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } - void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } - void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } - void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } - void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } - void pack_float(float d) { pack_float_impl(m_stream, d); } - void pack_double(double d) { pack_double_impl(m_stream, d); } - void pack_nil() { pack_nil_impl(m_stream); } - void pack_true() { pack_true_impl(m_stream); } - void pack_false() { pack_false_impl(m_stream); } - void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } - void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } - void pack_string(const char* b) { pack_string_impl(m_stream, b); } - void pack_raw(const char* b, size_t l) { pack_raw_impl(m_stream, (const void*)b, l); } - -private: - static void pack_int_impl(dynamic_stream& x, int d); - static void pack_unsigned_int_impl(dynamic_stream& x, unsigned int d); - static void pack_uint8_impl(dynamic_stream& x, uint8_t d); - static void pack_uint16_impl(dynamic_stream& x, uint16_t d); - static void pack_uint32_impl(dynamic_stream& x, uint32_t d); - static void pack_uint64_impl(dynamic_stream& x, uint64_t d); - static void pack_int8_impl(dynamic_stream& x, int8_t d); - static void pack_int16_impl(dynamic_stream& x, int16_t d); - static void pack_int32_impl(dynamic_stream& x, int32_t d); - static void pack_int64_impl(dynamic_stream& x, int64_t d); - static void pack_float_impl(dynamic_stream& x, float d); - static void pack_double_impl(dynamic_stream& x, double d); - static void pack_nil_impl(dynamic_stream& x); - static void pack_true_impl(dynamic_stream& x); - static void pack_false_impl(dynamic_stream& x); - static void pack_array_impl(dynamic_stream& x, unsigned int n); - static void pack_map_impl(dynamic_stream& x, unsigned int n); - static void pack_string_impl(dynamic_stream& x, const char* b); - static void pack_raw_impl(dynamic_stream& x, const void* b, size_t l); - static void append_buffer(dynamic_stream& x, const unsigned char* buf, unsigned int len) - { x.write((const char*)buf, len); } - -private: - dynamic_stream m_stream; - -private: - dynamic_packer(); -}; - -#define msgpack_pack_inline_func(name) \ - inline void dynamic_packer::pack_ ## name ## _impl -#define msgpack_pack_user dynamic_stream& -#define msgpack_pack_append_buffer append_buffer -#include "msgpack/pack_template.h" - -template -dynamic_packer::dynamic_packer(Stream& s) : m_stream(s) { } - -struct dynamic_stream::write_trampoline { -private: - template - struct ret_type { - typedef R (*type)(void*, const char*, size_t); - }; - - template - static R trampoline(void* obj, const char* buf, size_t len) - { - return (reinterpret_cast(obj)->*MemFun)(buf, len); - } - -public: - template - static typename ret_type::type get(R (Stream::*func)(Ptr*, Sz)) - { - R (*f)(void*, const char*, size_t) = - &trampoline; - return f; - } -}; - -template -dynamic_stream::dynamic_stream(Stream& s) -{ - m_object = reinterpret_cast(&s); - m_function = reinterpret_cast( - write_trampoline::get(&Stream::write) - ); -} - - -template -inline void pack(Stream& s, object o) -{ - dynamic_packer pk(s); - o.pack(pk); -} -*/ - - } // namespace msgpack #endif /* msgpack/pack.hpp */ diff --git a/cpp/type/raw.hpp b/cpp/type/raw.hpp index 14e52f4..e4f9dd3 100644 --- a/cpp/type/raw.hpp +++ b/cpp/type/raw.hpp @@ -82,7 +82,8 @@ inline std::string& operator>> (object o, std::string& v) template inline packer& operator<< (packer& o, const type::raw_ref& v) { - o.pack_raw(v.ptr, v.size); + o.pack_raw(v.size); + o.pack_raw_body(v.ptr, v.size); return o; } @@ -90,7 +91,8 @@ inline packer& operator<< (packer& o, const type::raw_ref& v) template inline packer& operator<< (packer& o, const std::string& v) { - o.pack_raw(v.data(), v.size()); + o.pack_raw(v.size()); + o.pack_raw_body(v.data(), v.size()); return o; } diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index ce39afd..cbf1356 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -257,7 +257,7 @@ zone* unpacker::release_zone() throw; } } - m_ctx->user(&*m_zone); + m_ctx->user(m_zone.get()); return old.release(); } @@ -268,7 +268,7 @@ object unpacker::data() void unpacker::reset() { - if(m_off != 0) { std::auto_ptr old(release_zone()); } + if(m_off != 0) { delete release_zone(); } m_ctx->reset(); } diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 69c7345..1dbf6fd 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -282,7 +282,7 @@ msgpack_pack_inline_func(map)(msgpack_pack_user x, unsigned int n) * Raw */ -msgpack_pack_inline_func(raw)(msgpack_pack_user x, const void* b, size_t l) +msgpack_pack_inline_func(raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; @@ -296,9 +296,12 @@ msgpack_pack_inline_func(raw)(msgpack_pack_user x, const void* b, size_t l) unsigned char buf[5] = {0xdb, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } - msgpack_pack_append_buffer(x, (const unsigned char*)b, l); } +msgpack_pack_inline_func(raw_body)(msgpack_pack_user x, const void* b, size_t l) +{ + msgpack_pack_append_buffer(x, (const unsigned char*)b, l); +} #undef msgpack_pack_inline_func #undef msgpack_pack_user diff --git a/ruby/pack.c b/ruby/pack.c index 54f610c..e62419d 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -45,7 +45,8 @@ static void msgpack_pack_true(VALUE x); static void msgpack_pack_false(VALUE x); static void msgpack_pack_array(VALUE x, unsigned int n); static void msgpack_pack_map(VALUE x, unsigned int n); -static void msgpack_pack_raw(VALUE x, const void* b, size_t l); +static void msgpack_pack_raw(VALUE x, size_t l); +static void msgpack_pack_raw_body(VALUE x, const void* b, size_t l); */ #include "msgpack/pack_template.h" @@ -123,7 +124,8 @@ static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self) static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); - msgpack_pack_raw(out, RSTRING_PTR(self), RSTRING_LEN(self)); + msgpack_pack_raw(out, RSTRING_LEN(self)); + msgpack_pack_raw_body(out, RSTRING_PTR(self), RSTRING_LEN(self)); return out; } From c3021d3236e1276bb575802597bfaf8279fced72 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0028/1172] lang/c/msgpack: added msgpack::define git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@75 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.hpp | 53 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 13402cc..5ed1fab 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -81,21 +81,6 @@ inline object& operator>> (object o, object& v) template packer& operator<< (packer& o, const object& v); - - -template -inline T& operator>> (object o, T& v) -{ - v.msgpack_unpack(o); - return v; -} - -template -inline packer& operator<< (packer& o, const T& v) -{ - o << v.msgpack_pack(); - return o; -} template @@ -123,6 +108,44 @@ inline void pack_copy(packer& o, T v) pack(o, v); } + + +template +inline T& operator>> (object o, T& v) +{ + v.msgpack_unpack(o); + return v; +} + +template +inline packer& operator<< (packer& o, const T& v) +{ + o << v.msgpack_pack(); + return o; +} + + +template +class define : public Type { +public: + typedef Type msgpack_type; + typedef define define_type; + + define() {} + define(msgpack_type v) : msgpack_type(v) {} + + msgpack_type msgpack_pack() const + { + return *this; + } + + void msgpack_unpack(object o) + { + convert(static_cast(*this), o); + } +}; + + template packer& operator<< (packer& o, const object& v) From ccdb39ac60f256db36a9a82b3792182f3b339d13 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0029/1172] lang/c/msgpack: msgpack::unpacker: fix buffer reallocation algorithm git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@76 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/unpack.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index cbf1356..ea4d414 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -198,7 +198,7 @@ void unpacker::expand_buffer(size_t len) { if(m_off == 0) { size_t next_size; - if(m_free != 0) { next_size = m_free * 2; } + if(m_used != 0) { next_size = (m_used + m_free) * 2; } else { next_size = UNPACKER_INITIAL_BUFFER_SIZE; } while(next_size < len + m_used) { next_size *= 2; } From 40ca91e10110ed7de761a04a96119d14adb41fff Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0030/1172] lang/c/msgpack: ./configure requires erb git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@77 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- configure.in | 1 - cpp/Makefile.am | 2 +- cpp/configure.in | 6 +++++- cpp/type/map.hpp | 1 + 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index 33acfa6..474d6e2 100644 --- a/configure.in +++ b/configure.in @@ -3,7 +3,6 @@ AM_INIT_AUTOMAKE(msgpack, 0.1.0) AC_CONFIG_HEADER(config.h) AC_PROG_LIBTOOL -#AC_CHECK_PROG(RUBY, ruby, ruby, [$PATH]) AC_CONFIG_SUBDIRS([c cpp]) AC_OUTPUT([Makefile]) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index e244763..61a616a 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -30,7 +30,7 @@ unpack.lo: msgpack/type/tuple.hpp zone.lo: msgpack/type/tuple.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb - erb $< > $@ + $(ERB) $< > $@ MOSTLYCLEANFILES = msgpack/type/tuple.hpp diff --git a/cpp/configure.in b/cpp/configure.in index a60a489..2c3e5d0 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -9,7 +9,11 @@ fi AC_PROG_CXX AC_PROG_LIBTOOL -AC_CHECK_PROG(ERB, erb, erb, [$PATH]) + +AC_CHECK_PROG(ERB, erb, erb) +if test "x$ERB" = x; then + AC_MSG_ERROR([cannot find erb. Ruby is needed to build.]) +fi AC_CHECK_LIB(stdc++, main) diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index 3b544df..c79f31c 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -21,6 +21,7 @@ #include "msgpack/object.hpp" #include #include +#include namespace msgpack { From bb4ea9d17c20007405b0b6e703f5f47bf54e5da0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0031/1172] lang/c/msgpack: improved configure.in git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@78 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/Makefile.am | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 61a616a..19c6007 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -30,7 +30,8 @@ unpack.lo: msgpack/type/tuple.hpp zone.lo: msgpack/type/tuple.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb - $(ERB) $< > $@ + $(ERB) $< > $@.tmp + mv $@.tmp $@ MOSTLYCLEANFILES = msgpack/type/tuple.hpp From d930090f130bf363d2330de1a15877ed9c5e9edc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0032/1172] lang/c/msgpack: C++ binding: abolished implicit convertion and added object::convert() git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@79 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.hpp | 61 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 5ed1fab..5f77d3e 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -44,8 +44,7 @@ namespace type { struct object { - unsigned char type; - union { + union union_type { bool boolean; uint64_t u64; int64_t i64; @@ -58,15 +57,36 @@ struct object { const char* ptr; uint32_t size; } ref; - } via; + }; - template - operator T() { T v; convert(v, *this); return v; }; - - template - T as() { T v; convert(v, *this); return v; } + unsigned char type; + union_type via; bool is_nil() { return type == type::NIL; } + + template + T as(); + + template + void convert(T& v); + +private: + struct implicit_type; + +public: + implicit_type convert(); +}; + + +struct object::implicit_type { + implicit_type(object o) : obj(o) { } + ~implicit_type() { } + + template + operator T() { return obj.as(); } + +private: + object obj; }; std::ostream& operator<< (std::ostream& s, const object o); @@ -74,9 +94,11 @@ std::ostream& operator<< (std::ostream& s, const object o); bool operator==(const object x, const object y); inline bool operator!=(const object x, const object y) { return !(x == y); } - inline object& operator>> (object o, object& v) - { v = o; return v; } +{ + v = o; + return v; +} template packer& operator<< (packer& o, const object& v); @@ -147,6 +169,25 @@ public: +inline object::implicit_type object::convert() +{ + return implicit_type(*this); +} + +template +inline T object::as() +{ + T v; + msgpack::convert(v, *this); + return v; +} + +template +void object::convert(T& v) +{ + msgpack::convert(v, *this); +} + template packer& operator<< (packer& o, const object& v) { From a0fd7c063adf9b249b40429b272178449a96f4ec Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0033/1172] lang/c/msgpack: wrote README git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@80 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- README | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/README b/README index 31a482a..87d9279 100644 --- a/README +++ b/README @@ -3,6 +3,56 @@ MessagePack Binary-based efficient data interchange format. +*Requirements + + MessagePack is only tested on Linux and Mac OS X, but it may run on other + UNIX-like platforms. + + Following programs is required to build: + - gcc >= 4.0 with C++ support + - ruby >= 1.8 (ruby is used as a preprocessor) + + +*Installation + + Simply run ./configure && make && make install to install C and C++ binding. + + $ ./configure --prefix=path/to/prefix + $ make + $ make install + + To install Ruby binding, run ./gengem.sh script in ruby/ directory and install + generated gem package. + + $ cd ruby + $ ./gengem.sh + $ gem install gem/pkg/msgpack-*.gem + + +*Usage + + C++: + #include + // TODO + + + C: + #include + /* TODO */ + + + Ruby: + require 'msgpack' + + # serialize + buf = [1, 2, 3].to_msgpack + + # deserialize + p MessagePack::unpack(buf) + + # TODO + + Copyright (C) 2008 FURUHASHI Sadayuki From 33986868162e0c312fd9547e7c05199d684bd07a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0034/1172] {c,cpp}/bootstrap: improve compatibility git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@81 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/bootstrap | 2 +- cpp/bootstrap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/c/bootstrap b/c/bootstrap index 6a1e814..7e61b82 100755 --- a/c/bootstrap +++ b/c/bootstrap @@ -1,3 +1,3 @@ #!/bin/sh NO_NEST=1 -source ../bootstrap +. ../bootstrap diff --git a/cpp/bootstrap b/cpp/bootstrap index 6a1e814..7e61b82 100755 --- a/cpp/bootstrap +++ b/cpp/bootstrap @@ -1,3 +1,3 @@ #!/bin/sh NO_NEST=1 -source ../bootstrap +. ../bootstrap From 8be25a7f3216007dd3e9561076bd2e7cf3945b90 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0035/1172] unpacker::unpacker() accepts initial buffer size git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@82 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/unpack.cpp | 9 +++++---- cpp/unpack.hpp | 11 +++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index ea4d414..8edf407 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -178,13 +178,14 @@ private: }; -unpacker::unpacker() : +unpacker::unpacker(size_t initial_buffer_size) : m_buffer(NULL), m_used(0), m_free(0), m_off(0), m_zone(new zone()), - m_ctx(new context(&*m_zone)) + m_ctx(new context(&*m_zone)), + m_initial_buffer_size(initial_buffer_size) { } @@ -199,14 +200,14 @@ void unpacker::expand_buffer(size_t len) if(m_off == 0) { size_t next_size; if(m_used != 0) { next_size = (m_used + m_free) * 2; } - else { next_size = UNPACKER_INITIAL_BUFFER_SIZE; } + else { next_size = m_initial_buffer_size; } while(next_size < len + m_used) { next_size *= 2; } m_buffer = m_zone->realloc(m_buffer, next_size); m_free = next_size - m_used; } else { - size_t next_size = UNPACKER_INITIAL_BUFFER_SIZE; + size_t next_size = m_initial_buffer_size; while(next_size < len + m_used - m_off) { next_size *= 2; } char* tmp = m_zone->malloc(next_size); diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index ffee60d..ac20de3 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -23,16 +23,13 @@ #include #include -#ifndef MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE -#define MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE 8*1024 +#ifndef MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE +#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE 8*1024 #endif namespace msgpack { -static const size_t UNPACKER_INITIAL_BUFFER_SIZE = MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE; - - struct unpack_error : public std::runtime_error { unpack_error(const std::string& msg) : std::runtime_error(msg) { } @@ -41,7 +38,7 @@ struct unpack_error : public std::runtime_error { class unpacker { public: - unpacker(); + unpacker(size_t initial_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE); ~unpacker(); public: @@ -139,6 +136,8 @@ private: struct context; context* m_ctx; + const size_t m_initial_buffer_size; + private: void expand_buffer(size_t len); From 05b63f6c87d45ea2e1aa8a4d9f8bf665a50591a5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0036/1172] C++: operator<< (std::ostream&, const object&): fix missing double support git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@83 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpp/object.cpp b/cpp/object.cpp index 45cfb5f..52b6ad0 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -41,6 +41,10 @@ std::ostream& operator<< (std::ostream& s, const object o) s << o.via.i64; break; + case type::DOUBLE: + s << o.via.dec; + break; + case type::RAW: (s << '"').write(o.via.ref.ptr, o.via.ref.size) << '"'; break; From 7e872371a241771489f3c48b7d364e4f6a9a248a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0037/1172] add examples git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@84 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- example/protocol.cc | 86 +++++++++++++++++++++++++++++ example/simple.cc | 31 +++++++++++ example/simple.rb | 5 ++ example/stream.cc | 131 ++++++++++++++++++++++++++++++++++++++++++++ example/stream.rb | 67 ++++++++++++++++++++++ 5 files changed, 320 insertions(+) create mode 100644 example/protocol.cc create mode 100644 example/simple.cc create mode 100644 example/simple.rb create mode 100644 example/stream.cc create mode 100644 example/stream.rb diff --git a/example/protocol.cc b/example/protocol.cc new file mode 100644 index 0000000..e7f2a38 --- /dev/null +++ b/example/protocol.cc @@ -0,0 +1,86 @@ +#include +#include +#include +#include + +namespace myprotocol { + using namespace msgpack::type; + using msgpack::define; + + struct Get : define< tuple > { + Get() { } + Get(uint32_t f, const std::string& k) : + define_type(msgpack_type(f, k)) { } + uint32_t& flags() { return get<0>(); } + std::string& key() { return get<1>(); } + }; + + struct Put : define< tuple > { + Put() { } + Put(uint32_t f, const std::string& k, const char* valref, size_t vallen) : + define_type(msgpack_type( f, k, raw_ref(valref,vallen) )) { } + uint32_t& flags() { return get<0>(); } + std::string& key() { return get<1>(); } + raw_ref& value() { return get<2>(); } + }; + + struct MultiGet : define< std::vector > { + }; +} + + +int main(void) +{ + // send Get request + std::stringstream stream; + { + myprotocol::Get req; + req.flags() = 0; + req.key() = "key0"; + msgpack::pack(stream, req); + } + + stream.seekg(0); + + // receive Get request + { + std::string buffer(stream.str()); + + msgpack::zone mempool; + msgpack::object o = + msgpack::unpack(buffer.data(), buffer.size(), mempool); + + myprotocol::Get req; + msgpack::convert(req, o); + std::cout << "received: " << o << std::endl; + } + + + stream.str(""); + + + // send MultiGet request + { + myprotocol::MultiGet req; + req.push_back( myprotocol::Get(1, "key1") ); + req.push_back( myprotocol::Get(2, "key2") ); + req.push_back( myprotocol::Get(3, "key3") ); + msgpack::pack(stream, req); + } + + stream.seekg(0); + + // receive MultiGet request + { + std::string buffer(stream.str()); + + msgpack::zone mempool; + msgpack::object o = + msgpack::unpack(buffer.data(), buffer.size(), mempool); + + myprotocol::MultiGet req; + msgpack::convert(req, o); + std::cout << "received: " << o << std::endl; + } +} + diff --git a/example/simple.cc b/example/simple.cc new file mode 100644 index 0000000..74beeae --- /dev/null +++ b/example/simple.cc @@ -0,0 +1,31 @@ +#include +#include +#include +#include + +int main(void) +{ + // this is target object + msgpack::type::tuple src(1, true, "example"); + + // any classes that implements write(const char*,size_t) can be a buffer + std::stringstream buffer; + msgpack::pack(buffer, src); + + // send the buffer ... + buffer.seekg(0); + + // deserialize the buffer into msgpack::object type + msgpack::zone mempool; + std::string str(buffer.str()); + msgpack::object deserialized = + msgpack::unpack(str.data(), str.size(), mempool); + + // msgpack::object supports ostream + std::cout << deserialized << std::endl; + + // convert msgpack::object type into the original type + msgpack::type::tuple dst; + msgpack::convert(dst, deserialized); +} + diff --git a/example/simple.rb b/example/simple.rb new file mode 100644 index 0000000..90b4696 --- /dev/null +++ b/example/simple.rb @@ -0,0 +1,5 @@ +require 'msgpack' + +serialized = [1, -1, true, false, nil, {"key" => "value"}].to_msgpack +p MessagePack.unpack(serialized) + diff --git a/example/stream.cc b/example/stream.cc new file mode 100644 index 0000000..49927de --- /dev/null +++ b/example/stream.cc @@ -0,0 +1,131 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +class Server { +public: + Server(int sock) : m_sock(sock) { } + ~Server() { } + + typedef std::auto_ptr auto_zone; + + void socket_readable() + { + m_pac.reserve_buffer(1024); + + ssize_t count = + read(m_sock, m_pac.buffer(), m_pac.buffer_capacity()); + + if(count < 0) { + if(errno == EAGAIN || errno == EINTR) { + return; + } else { + throw std::runtime_error(strerror(errno)); + } + } else if(count == 0) { + throw std::runtime_error("connection closed"); + } + + m_pac.buffer_consumed(count); + + while(m_pac.execute()) { + msgpack::object msg = m_pac.data(); + + auto_zone life( m_pac.release_zone() ); + + m_pac.reset(); + + process_message(msg, life); + } + } + +private: + void process_message(msgpack::object msg, auto_zone& life) + { + std::cout << "message reached: " << msg << std::endl; + } + +private: + int m_sock; + msgpack::unpacker m_pac; +}; + + +static void* run_server(void* arg) +try { + Server* srv = reinterpret_cast(arg); + + while(true) { + srv->socket_readable(); + } + return NULL; + +} catch (std::exception& e) { + std::cerr << "error while processing client packet: " + << e.what() << std::endl; + return NULL; + +} catch (...) { + std::cerr << "error while processing client packet: " + << "unknown error" << std::endl; + return NULL; +} + + +struct fwriter { + fwriter(int fd) : m_fp( fdopen(fd, "w") ) { } + + void write(const char* buf, size_t buflen) + { + size_t count = fwrite(buf, buflen, 1, m_fp); + //if(fwrite(buf, buflen, 1, m_fp) < 1) { + if(count < 1) { + std::cout << buflen << std::endl; + std::cout << count << std::endl; + throw std::runtime_error(strerror(errno)); + } + } + + void flush() { fflush(m_fp); } + + void close() { fclose(m_fp); } + +private: + FILE* m_fp; +}; + + +int main(void) +{ + int pair[2]; + pipe(pair); + + // run server thread + Server srv(pair[0]); + pthread_t thread; + pthread_create(&thread, NULL, + run_server, reinterpret_cast(&srv)); + + // client thread: + fwriter writer(pair[1]); + + typedef msgpack::type::tuple put_t; + typedef msgpack::type::tuple get_t; + + put_t req1("put", "apple", "red"); + put_t req2("put", "lemon", "yellow"); + get_t req3("get", "apple"); + msgpack::pack(writer, req1); + msgpack::pack(writer, req2); + msgpack::pack(writer, req3); + writer.flush(); + writer.close(); + + pthread_join(thread, NULL); +} + diff --git a/example/stream.rb b/example/stream.rb new file mode 100644 index 0000000..e53ce82 --- /dev/null +++ b/example/stream.rb @@ -0,0 +1,67 @@ +require 'msgpack' + +class Server + def initialize(sock) + @sock = sock + @pk = MessagePack::Unpacker.new + @buffer = '' + @nread = 0 + end + + def run + while true + begin + data = @sock.sysread(1024) + rescue + puts "connection closed (#{$!})" + return + end + receive_data(data) + end + end + + private + def receive_data(data) + @buffer << data + + while true + @nread = @pk.execute(@buffer, @nread) + + if @pk.finished? + msg = @pk.data + process_message(msg) + + @pk.reset + @buffer.slice!(0, @nread) + @nread = 0 + next unless @buffer.empty? + end + + break + end + + rescue + puts "error while processing client packet: #{$!}" + end + + def process_message(msg) + puts "message reached: #{msg.inspect}" + end +end + + +rpipe, wpipe = IO.pipe + +# run server thread +thread = Thread.new(Server.new(rpipe)) {|srv| + srv.run +} + +# client thread: +wpipe.write ["put", "apple", "red"].to_msgpack +wpipe.write ["put", "lemon", "yellow"].to_msgpack +wpipe.write ["get", "apple"].to_msgpack +wpipe.close + +thread.join + From 92dd6de59da206571ce488fe1daae80afa5b6b13 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0038/1172] version 0.2.0 git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@85 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 474d6e2..bd34885 100644 --- a/configure.in +++ b/configure.in @@ -1,5 +1,5 @@ AC_INIT(msgpack/unpack_template.h) -AM_INIT_AUTOMAKE(msgpack, 0.1.0) +AM_INIT_AUTOMAKE(msgpack, 0.2.0) AC_CONFIG_HEADER(config.h) AC_PROG_LIBTOOL From 512e7600f4977e7d4369af5b0d0f0c540427833a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0039/1172] msgpack::type::tuple supports reference element git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@86 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/type/tuple.hpp.erb | 95 ++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 55 deletions(-) diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index 397f660..13d4bd7 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -32,36 +32,48 @@ template , typename A<%=i% struct tuple; template -struct tuple_type; +struct tuple_element; template -struct const_tuple_type; +struct const_tuple_element; template -struct tuple_element { +struct tuple_type { typedef T type; - tuple_element(T& x) : _x(x) {} - type& get() { return _x; } - const type& get() const { return _x; } -private: - type& _x; + typedef T value_type; + typedef T& reference; + typedef const T& const_reference; + typedef const T& transparent_reference; }; template -struct const_tuple_element { +struct tuple_type { typedef T type; - const_tuple_element(const T& x) : _x(x) {} - const type& get() const { return _x; } -private: - const type& _x; + typedef T& value_type; + typedef T& reference; + typedef const T& const_reference; + typedef T& transparent_reference; +}; + +template +struct tuple_type { + typedef T type; + typedef T& value_type; + typedef T& reference; + typedef const T& const_reference; + typedef const T& transparent_reference; }; <%0.upto(GENERATION_LIMIT) {|i|%> <%0.upto(i) {|j|%> template , typename A<%=k%><%}%>> -struct tuple_type, A<%=k%><%}%>>, <%=j%>> : tuple_element> { - tuple_type(tuple, A<%=k%> <%}%>>& x) : tuple_element>(x.a<%=j%>) {} +struct tuple_element, A<%=k%><%}%>>, <%=j%>> : tuple_type> { + tuple_element(tuple, A<%=k%> <%}%>>& x) : _x(x.a<%=j%>) {} + typename tuple_type>::reference get() { return _x; } + typename tuple_type>::const_reference get() const { return _x; } +private: + typename tuple_type>::reference _x; }; <%}%> <%}%> @@ -70,8 +82,11 @@ struct tuple_type, A<%=k%><%}%>>, <%=j%>> : tuple_ele <%0.upto(GENERATION_LIMIT) {|i|%> <%0.upto(i) {|j|%> template , typename A<%=k%><%}%>> -struct const_tuple_type, A<%=k%><%}%>>, <%=j%>> : const_tuple_element> { - const_tuple_type(const tuple, A<%=k%><%}%>>& x) : const_tuple_element>(x.a<%=j%>) {} +struct const_tuple_element, A<%=k%><%}%>>, <%=j%>> : tuple_type> { + const_tuple_element(const tuple, A<%=k%><%}%>>& x) : _x(x.a<%=j%>) {} + typename tuple_type>::const_reference get() const { return _x; } +private: + typename tuple_type>::const_reference _x; }; <%}%> <%}%> @@ -79,7 +94,7 @@ struct const_tuple_type, A<%=k%><%}%>>, <%=j%>> : con <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> -tuple, A<%=j%><%}%>> make_tuple(const A0& a0<%1.upto(i) {|j|%>, const A<%=j%>& a<%=j%><%}%>) +tuple, A<%=j%><%}%>> make_tuple(typename tuple_type::transparent_reference a0<%1.upto(i) {|j|%>, typename tuple_type>::transparent_reference a<%=j%><%}%>) { return tuple, A<%=j%><%}%>>(a0<%1.upto(i) {|j|%>, a<%=j%><%}%>); } @@ -92,14 +107,15 @@ struct tuple<> { <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> struct tuple, A<%=j%><%}%>> { + typedef tuple, A<%=j%><%}%>> value_type; tuple() {} - tuple(const A0& _a0<%1.upto(i) {|j|%>, const A<%=j%>& _a<%=j%><%}%>) : - a0(_a0) <%1.upto(i) {|j|%>, a<%=j%>(_a<%=j%>)<%}%> {} + tuple(typename tuple_type::transparent_reference _a0<%1.upto(i) {|j|%>, typename tuple_type>::transparent_reference _a<%=j%><%}%>) : + a0(_a0)<%1.upto(i) {|j|%>, a<%=j%>(_a<%=j%>)<%}%> {} tuple(object o) { convert(*this, o); } - template typename tuple_type, A<%=j%><%}%>>, N>::type& get() - { return tuple_type, A<%=j%><%}%>>, N>(*this).get(); } - template const typename const_tuple_type, A<%=j%><%}%>>, N>::type& get() const - { return const_tuple_type, A<%=j%><%}%>>, N>(*this).get(); } + template typename tuple_element::reference get() + { return tuple_element(*this).get(); } + template typename const_tuple_element::const_reference get() const + { return const_tuple_element(*this).get(); } <%0.upto(i) {|j|%> A<%=j%> a<%=j%>;<%}%> }; @@ -148,37 +164,6 @@ const packer& operator<< ( <%}%> -// FIXME -/* -template -struct tuple_just; - -template -struct tuple_just { - A0 a0; - static inline void convert(object o, tuple_just& v) - { - if(o.type != type::ARRAY) { throw type_error(); } - if(o.v.container.size != 1) { throw type_error(); } - msgpack::convert(o.v.container.ptr[0], v.a0); - } -}; - -template -struct tuple_just { - A0 a0; - A1 a1; - static inline void convert(object o, tuple_just& v) - { - if(o.type != type::ARRAY) { throw type_error(); } - if(o.v.container.size != 2) { throw type_error(); } - msgpack::convert(o.v.container.ptr[0], v.a0); - msgpack::convert(o.v.container.ptr[1], v.a1); - } -}; -*/ - - } // namespace msgpack #endif /* msgpack/type/tuple.hpp */ From 15837bc3322b6f2fd6191187f4754f7d65c786a4 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0040/1172] add ruby/test_case.rb git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@87 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- ruby/test_case.rb | 142 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 ruby/test_case.rb diff --git a/ruby/test_case.rb b/ruby/test_case.rb new file mode 100644 index 0000000..af6d160 --- /dev/null +++ b/ruby/test_case.rb @@ -0,0 +1,142 @@ +require 'test/unit' +require 'msgpack' + +class MessagePackTestFormat < Test::Unit::TestCase + def self.it(name, &block) + define_method("test_#{name}", &block) + end + + it "nil" do + check 1, nil + end + + it "true" do + check 1, true + end + + it "false" do + check 1, false + end + + it "zero" do + check 1, 0 + end + + it "positive fixnum" do + check 1, 1 + check 1, (1<<6) + check 1, (1<<7)-1 + end + + it "positive int 8" do + check 1, -1 + check 2, (1<<7) + check 2, (1<<8)-1 + end + + it "positive int 16" do + check 3, (1<<8) + check 3, (1<<16)-1 + end + + it "positive int 32" do + check 5, (1<<16) + check 5, (1<<32)-1 + end + + it "positive int 64" do + check 9, (1<<32) + check 9, (1<<64)-1 + end + + it "negative fixnum" do + check 1, -1 + check 1, -((1<<5)-1) + check 1, -(1<<5) + end + + it "negative int 8" do + check 2, -((1<<5)+1) + check 2, -(1<<7) + end + + it "negative int 16" do + check 3, -((1<<7)+1) + check 3, -(1<<15) + end + + it "negative int 32" do + check 5, -((1<<15)+1) + check 5, -(1<<31) + end + + it "negative int 64" do + check 9, -((1<<31)+1) + check 9, -(1<<63) + end + + it "fixraw" do + check_raw 1, 0 + check_raw 1, (1<<5)-1 + end + + it "raw 16" do + check_raw 3, (1<<5) + check_raw 3, (1<<16)-1 + end + + it "raw 32" do + check_raw 5, (1<<16) + #check_raw 5, (1<<32)-1 # memory error + end + + it "fixarray" do + check_array 1, 0 + check_array 1, (1<<4)-1 + end + + it "array 16" do + check_array 3, (1<<4) + check_array 3, (1<<16)-1 + end + + it "array 32" do + check_array 5, (1<<16) + #check_array 5, (1<<32)-1 # memory error + end + +# it "fixmap" do +# check_map 1, 0 +# check_map 1, (1<<4)-1 +# end +# +# it "map 16" do +# check_map 3, (1<<4) +# check_map 3, (1<<16)-1 +# end +# +# it "map 32" do +# check_map 5, (1<<16) +# #check_map 5, (1<<32)-1 # memory error +# end + + private + def check(len, obj) + v = obj.to_msgpack + assert_equal(v.length, len) + assert_equal(MessagePack.unpack(v), obj) + end + + def check_raw(overhead, num) + check num+overhead, " "*num + end + + def check_array(overhead, num) + check num+overhead, Array.new(num) + end + + def check_map(overhead, num) + # FIXME + end +end + From 8f3444c08141520a0977adce643ce0eb7f0324cd Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0041/1172] ruby binding: fix Fixnum serialization bug on x86_64 git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@88 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- configure.in | 2 +- msgpack/pack_template.h | 141 +++++++++++++++++++++++++++++--- ruby/gem/lib/msgpack/version.rb | 2 +- ruby/pack.c | 6 +- 4 files changed, 134 insertions(+), 17 deletions(-) diff --git a/configure.in b/configure.in index bd34885..63762b0 100644 --- a/configure.in +++ b/configure.in @@ -1,5 +1,5 @@ AC_INIT(msgpack/unpack_template.h) -AM_INIT_AUTOMAKE(msgpack, 0.2.0) +AM_INIT_AUTOMAKE(msgpack, 0.2.1) AC_CONFIG_HEADER(config.h) AC_PROG_LIBTOOL diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 1dbf6fd..04f8c98 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -68,15 +68,14 @@ * Integer */ -// wrapper -msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) +static inline void msgpack_pack_compress_int32(msgpack_pack_user x, uint32_t d) { - if(d < -32) { - if(d < -32768) { + if(d < -(1<<5)) { + if(d < -(1<<15)) { // signed 32 const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); - } else if(d < -128) { + } else if(d < -(1<<7)) { // signed 16 const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); @@ -85,15 +84,15 @@ msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) const unsigned char buf[2] = {0xd0, (uint8_t)d}; msgpack_pack_append_buffer(x, buf, 2); } - } else if(d < 128) { + } else if(d < (1<<7)) { // fixnum msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); } else { - if(d < 256) { + if(d < (1<<8)) { // unsigned 8 const unsigned char buf[2] = {0xcc, (uint8_t)d}; msgpack_pack_append_buffer(x, buf, 2); - } else if(d < 65536) { + } else if(d < (1<<16)) { // unsigned 16 const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); @@ -105,11 +104,10 @@ msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) } } -// wrapper -msgpack_pack_inline_func(unsigned_int)(msgpack_pack_user x, unsigned int d) +static inline void msgpack_pack_compress_uint32(msgpack_pack_user x, uint32_t d) { - if(d < 256) { - if(d < 128) { + if(d < (1<<8)) { + if(d < (1<<7)) { // fixnum msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); } else { @@ -118,7 +116,7 @@ msgpack_pack_inline_func(unsigned_int)(msgpack_pack_user x, unsigned int d) msgpack_pack_append_buffer(x, buf, 2); } } else { - if(d < 65536) { + if(d < (1<<16)) { // unsigned 16 const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); @@ -130,6 +128,123 @@ msgpack_pack_inline_func(unsigned_int)(msgpack_pack_user x, unsigned int d) } } +static inline void msgpack_pack_compress_int64(msgpack_pack_user x, int64_t d) +{ + if(d < -(1LL<<5)) { + if(d < -(1LL<<15)) { + if(d < -(1LL<<31)) { + // signed 64 + const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); + } else { + // signed 32 + const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } + } else { + if(d < -(1<<7)) { + // signed 16 + const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + // signed 8 + const unsigned char buf[2] = {0xd0, (uint8_t)d}; + msgpack_pack_append_buffer(x, buf, 2); + } + } + } else if(d < (1<<7)) { + // fixnum + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); + } else { + if(d < (1LL<<16)) { + if(d < (1<<8)) { + // unsigned 8 + const unsigned char buf[2] = {0xcc, (uint8_t)d}; + msgpack_pack_append_buffer(x, buf, 2); + } else { + // unsigned 16 + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } + } else { + if(d < (1LL<<32)) { + // unsigned 32 + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } else { + // unsigned 64 + const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); + } + } + } +} + +static inline void msgpack_pack_compress_uint64(msgpack_pack_user x, uint64_t d) +{ + if(d < (1ULL<<8)) { + if(d < (1<<7)) { + // fixnum + msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); + } else { + // unsigned 8 + const unsigned char buf[2] = {0xcc, (uint8_t)d}; + msgpack_pack_append_buffer(x, buf, 2); + } + } else { + if(d < (1ULL<<16)) { + // signed 16 + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else if(d < (1ULL<<32)) { + // signed 32 + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } else { + // signed 64 + const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); + } + } +} + +msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) +{ +#if SIZEOF_INT == 8 + msgpack_pack_compress_int64(x, d); +#else + msgpack_pack_compress_int32(x, d); +#endif +} + +msgpack_pack_inline_func(unsigned_int)(msgpack_pack_user x, unsigned int d) +{ +#if SIZEOF_INT == 8 + msgpack_pack_compress_uint64(x, d); +#else + msgpack_pack_compress_uint32(x, d); +#endif +} + +msgpack_pack_inline_func(long)(msgpack_pack_user x, long d) +{ +#if SIZEOF_LONG == 8 + msgpack_pack_compress_int64(x, d); +#else + msgpack_pack_compress_int32(x, d); +#endif +} + +msgpack_pack_inline_func(unsigned_long)(msgpack_pack_user x, unsigned long d) +{ +#if SIZEOF_LONG == 8 + msgpack_pack_compress_uint64(x, d); +#else + msgpack_pack_compress_uint32(x, d); +#endif +} + + msgpack_pack_inline_func(uint8)(msgpack_pack_user x, uint8_t d) { if(d < 128) { diff --git a/ruby/gem/lib/msgpack/version.rb b/ruby/gem/lib/msgpack/version.rb index b2a5db6..229c746 100644 --- a/ruby/gem/lib/msgpack/version.rb +++ b/ruby/gem/lib/msgpack/version.rb @@ -2,7 +2,7 @@ module MessagePack module VERSION #:nodoc: MAJOR = 0 MINOR = 2 - TINY = 0 + TINY = 1 STRING = [MAJOR, MINOR, TINY].join('.') end diff --git a/ruby/pack.c b/ruby/pack.c index e62419d..5262024 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -20,7 +20,7 @@ #include #define msgpack_pack_inline_func(name) \ - static void msgpack_pack_##name + static inline void msgpack_pack_##name #define msgpack_pack_user VALUE @@ -30,6 +30,8 @@ /* static void msgpack_pack_int(VALUE x, int d); static void msgpack_pack_unsigned_int(VALUE x, unsigned int d); +static void msgpack_pack_long(VALUE x, long d); +static void msgpack_pack_unsigned_long(VALUE x, unsigned long d); static void msgpack_pack_uint8(VALUE x, uint8_t d); static void msgpack_pack_uint16(VALUE x, uint16_t d); static void msgpack_pack_uint32(VALUE x, uint32_t d); @@ -93,7 +95,7 @@ static VALUE MessagePack_FalseClass_to_msgpack(int argc, VALUE *argv, VALUE self static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); - msgpack_pack_int(out, FIX2INT(self)); + msgpack_pack_long(out, FIX2LONG(self)); return out; } From adba617f45f89cacbd23667744d0cc17668ecdda Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0042/1172] c, c++ binding: catch up with ruby binding git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@89 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/pack.h | 2 + cpp/pack.hpp | 42 +++--- msgpack/pack_template.h | 276 ++++++++++++++++++++-------------------- 3 files changed, 165 insertions(+), 155 deletions(-) diff --git a/c/pack.h b/c/pack.h index ec0683e..fac642f 100644 --- a/c/pack.h +++ b/c/pack.h @@ -39,6 +39,8 @@ void msgpack_pack_free(msgpack_pack_t* ctx); void msgpack_pack_int(msgpack_pack_t* ctx, int d); void msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); +void msgpack_pack_long(msgpack_pack_t* ctx, long d); +void msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); void msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); void msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); void msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 4fda4ff..aee4b87 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -30,29 +30,33 @@ public: packer(Stream& s); public: - void pack_int(int d) { pack_int_impl(m_stream, d); } - void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } - void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } - void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } - void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } - void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } - void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } - void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } - void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } - void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } - void pack_float(float d) { pack_float_impl(m_stream, d); } - void pack_double(double d) { pack_double_impl(m_stream, d); } - void pack_nil() { pack_nil_impl(m_stream); } - void pack_true() { pack_true_impl(m_stream); } - void pack_false() { pack_false_impl(m_stream); } - void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } - void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } - void pack_raw(size_t l) { pack_raw_impl(m_stream, l); } - void pack_raw_body(const char* b, size_t l) { pack_raw_body_impl(m_stream, b, l); } + void pack_int(int d) { pack_int_impl(m_stream, d); } + void pack_long(long d) { pack_long_impl(m_stream, d); } + void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } + void pack_unsigned_long(unsigned long d) { pack_unsigned_long_impl(m_stream, d); } + void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } + void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } + void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } + void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } + void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } + void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } + void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } + void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } + void pack_float(float d) { pack_float_impl(m_stream, d); } + void pack_double(double d) { pack_double_impl(m_stream, d); } + void pack_nil() { pack_nil_impl(m_stream); } + void pack_true() { pack_true_impl(m_stream); } + void pack_false() { pack_false_impl(m_stream); } + void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } + void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } + void pack_raw(size_t l) { pack_raw_impl(m_stream, l); } + void pack_raw_body(const char* b, size_t l) { pack_raw_body_impl(m_stream, b, l); } private: static void pack_int_impl(Stream& x, int d); + static void pack_long_impl(Stream& x, long d); static void pack_unsigned_int_impl(Stream& x, unsigned int d); + static void pack_unsigned_long_impl(Stream& x, unsigned long d); static void pack_uint8_impl(Stream& x, uint8_t d); static void pack_uint16_impl(Stream& x, uint16_t d); static void pack_uint32_impl(Stream& x, uint32_t d); diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 04f8c98..94fbfd4 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -68,145 +68,145 @@ * Integer */ -static inline void msgpack_pack_compress_int32(msgpack_pack_user x, uint32_t d) -{ - if(d < -(1<<5)) { - if(d < -(1<<15)) { - // signed 32 - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; - msgpack_pack_append_buffer(x, buf, 5); - } else if(d < -(1<<7)) { - // signed 16 - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; - msgpack_pack_append_buffer(x, buf, 3); - } else { - // signed 8 - const unsigned char buf[2] = {0xd0, (uint8_t)d}; - msgpack_pack_append_buffer(x, buf, 2); - } - } else if(d < (1<<7)) { - // fixnum - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); - } else { - if(d < (1<<8)) { - // unsigned 8 - const unsigned char buf[2] = {0xcc, (uint8_t)d}; - msgpack_pack_append_buffer(x, buf, 2); - } else if(d < (1<<16)) { - // unsigned 16 - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; - msgpack_pack_append_buffer(x, buf, 3); - } else { - // unsigned 32 - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; - msgpack_pack_append_buffer(x, buf, 5); - } - } -} +#define msgpack_pack_compress_int32(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<15)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) -static inline void msgpack_pack_compress_uint32(msgpack_pack_user x, uint32_t d) -{ - if(d < (1<<8)) { - if(d < (1<<7)) { - // fixnum - msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); - } else { - // unsigned 8 - const unsigned char buf[2] = {0xcc, (uint8_t)d}; - msgpack_pack_append_buffer(x, buf, 2); - } - } else { - if(d < (1<<16)) { - // unsigned 16 - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; - msgpack_pack_append_buffer(x, buf, 3); - } else { - // unsigned 32 - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; - msgpack_pack_append_buffer(x, buf, 5); - } - } -} +#define msgpack_pack_compress_uint32(x, d) \ +do { \ + if(d < (1<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) -static inline void msgpack_pack_compress_int64(msgpack_pack_user x, int64_t d) -{ - if(d < -(1LL<<5)) { - if(d < -(1LL<<15)) { - if(d < -(1LL<<31)) { - // signed 64 - const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; - msgpack_pack_append_buffer(x, buf, 9); - } else { - // signed 32 - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; - msgpack_pack_append_buffer(x, buf, 5); - } - } else { - if(d < -(1<<7)) { - // signed 16 - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; - msgpack_pack_append_buffer(x, buf, 3); - } else { - // signed 8 - const unsigned char buf[2] = {0xd0, (uint8_t)d}; - msgpack_pack_append_buffer(x, buf, 2); - } - } - } else if(d < (1<<7)) { - // fixnum - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); - } else { - if(d < (1LL<<16)) { - if(d < (1<<8)) { - // unsigned 8 - const unsigned char buf[2] = {0xcc, (uint8_t)d}; - msgpack_pack_append_buffer(x, buf, 2); - } else { - // unsigned 16 - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; - msgpack_pack_append_buffer(x, buf, 3); - } - } else { - if(d < (1LL<<32)) { - // unsigned 32 - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; - msgpack_pack_append_buffer(x, buf, 5); - } else { - // unsigned 64 - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; - msgpack_pack_append_buffer(x, buf, 9); - } - } - } -} +#define msgpack_pack_compress_int64(x, d) \ +do { \ + if(d < -(1LL<<5)) { \ + if(d < -(1LL<<15)) { \ + if(d < -(1LL<<31)) { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } else { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } else { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } else { \ + if(d < (1LL<<16)) { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } else { \ + if(d < (1LL<<32)) { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* unsigned 64 */ \ + const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ + } \ +} while(0) -static inline void msgpack_pack_compress_uint64(msgpack_pack_user x, uint64_t d) -{ - if(d < (1ULL<<8)) { - if(d < (1<<7)) { - // fixnum - msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); - } else { - // unsigned 8 - const unsigned char buf[2] = {0xcc, (uint8_t)d}; - msgpack_pack_append_buffer(x, buf, 2); - } - } else { - if(d < (1ULL<<16)) { - // signed 16 - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; - msgpack_pack_append_buffer(x, buf, 3); - } else if(d < (1ULL<<32)) { - // signed 32 - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; - msgpack_pack_append_buffer(x, buf, 5); - } else { - // signed 64 - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; - msgpack_pack_append_buffer(x, buf, 9); - } - } -} +#define msgpack_pack_compress_uint64(x, d) \ +do { \ + if(d < (1ULL<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1ULL<<16)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else if(d < (1ULL<<32)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ +} while(0) msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) { @@ -244,6 +244,10 @@ msgpack_pack_inline_func(unsigned_long)(msgpack_pack_user x, unsigned long d) #endif } +#undef msgpack_pack_compress_int32 +#undef msgpack_pack_compress_uint32 +#undef msgpack_pack_compress_int64 +#undef msgpack_pack_compress_uint64 msgpack_pack_inline_func(uint8)(msgpack_pack_user x, uint8_t d) { From cd973b8483af21d4a3a03497aa223b82b5c1a9a2 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0043/1172] integrate machine-dependent integer serialization routine to msgpack/pack_template.h git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@90 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- Makefile.am | 1 + c/pack.c | 4 + c/pack.h | 6 + cpp/pack.hpp | 74 +++++-- cpp/type/integer.hpp | 116 +--------- msgpack/pack_define.h | 26 +++ msgpack/pack_template.h | 481 ++++++++++++++++++++++++++++++---------- ruby/gem/Manifest.txt | 1 + ruby/gengem.sh | 1 + ruby/pack.c | 30 +-- 10 files changed, 468 insertions(+), 272 deletions(-) create mode 100644 msgpack/pack_define.h diff --git a/Makefile.am b/Makefile.am index d79f7eb..c23320a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,6 +1,7 @@ SUBDIRS = c cpp nobase_include_HEADERS = \ + msgpack/pack_define.h \ msgpack/pack_template.h \ msgpack/unpack_define.h \ msgpack/unpack_template.h diff --git a/c/pack.c b/c/pack.c index 766a9d1..f4787c0 100644 --- a/c/pack.c +++ b/c/pack.c @@ -16,12 +16,16 @@ * limitations under the License. */ #include "msgpack/pack.h" +#include "msgpack/pack_define.h" #include #define msgpack_pack_inline_func(name) \ void msgpack_pack_##name +#define msgpack_pack_inline_func_cint(name) \ + void msgpack_pack_##name + #define msgpack_pack_user msgpack_pack_t* #define msgpack_pack_append_buffer(user, buf, len) \ diff --git a/c/pack.h b/c/pack.h index fac642f..ecf675c 100644 --- a/c/pack.h +++ b/c/pack.h @@ -41,6 +41,7 @@ void msgpack_pack_int(msgpack_pack_t* ctx, int d); void msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); void msgpack_pack_long(msgpack_pack_t* ctx, long d); void msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); + void msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); void msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); void msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); @@ -49,13 +50,18 @@ void msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); void msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); void msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); void msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); + void msgpack_pack_float(msgpack_pack_t* ctx, float d); void msgpack_pack_double(msgpack_pack_t* ctx, double d); + void msgpack_pack_nil(msgpack_pack_t* ctx); void msgpack_pack_true(msgpack_pack_t* ctx); void msgpack_pack_false(msgpack_pack_t* ctx); + void msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); + void msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); + void msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); void msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); diff --git a/cpp/pack.hpp b/cpp/pack.hpp index aee4b87..e860f4c 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -20,6 +20,8 @@ #include // __BYTE_ORDER #include +#include +#include "msgpack/pack_define.h" namespace msgpack { @@ -30,33 +32,38 @@ public: packer(Stream& s); public: - void pack_int(int d) { pack_int_impl(m_stream, d); } - void pack_long(long d) { pack_long_impl(m_stream, d); } - void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } - void pack_unsigned_long(unsigned long d) { pack_unsigned_long_impl(m_stream, d); } - void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } - void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } - void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } - void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } - void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } - void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } - void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } - void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } - void pack_float(float d) { pack_float_impl(m_stream, d); } - void pack_double(double d) { pack_double_impl(m_stream, d); } - void pack_nil() { pack_nil_impl(m_stream); } - void pack_true() { pack_true_impl(m_stream); } - void pack_false() { pack_false_impl(m_stream); } - void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } - void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } + void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } + void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } + void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } + void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } + void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } + void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } + void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } + void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } + + void pack_short(int d) { pack_short_impl(m_stream, d); } + void pack_int(int d) { pack_int_impl(m_stream, d); } + void pack_long(long d) { pack_long_impl(m_stream, d); } + void pack_long_long(long long d) { pack_long_long_impl(m_stream, d); } + void pack_unsigned_short(unsigned short d) { pack_unsigned_short_impl(m_stream, d); } + void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } + void pack_unsigned_long_long(unsigned long long d) { pack_unsigned_long_long_impl(m_stream, d); } + + void pack_float(float d) { pack_float_impl(m_stream, d); } + void pack_double(double d) { pack_double_impl(m_stream, d); } + + void pack_nil() { pack_nil_impl(m_stream); } + void pack_true() { pack_true_impl(m_stream); } + void pack_false() { pack_false_impl(m_stream); } + + void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } + + void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } + void pack_raw(size_t l) { pack_raw_impl(m_stream, l); } void pack_raw_body(const char* b, size_t l) { pack_raw_body_impl(m_stream, b, l); } private: - static void pack_int_impl(Stream& x, int d); - static void pack_long_impl(Stream& x, long d); - static void pack_unsigned_int_impl(Stream& x, unsigned int d); - static void pack_unsigned_long_impl(Stream& x, unsigned long d); static void pack_uint8_impl(Stream& x, uint8_t d); static void pack_uint16_impl(Stream& x, uint16_t d); static void pack_uint32_impl(Stream& x, uint32_t d); @@ -65,13 +72,27 @@ private: static void pack_int16_impl(Stream& x, int16_t d); static void pack_int32_impl(Stream& x, int32_t d); static void pack_int64_impl(Stream& x, int64_t d); + + static void pack_short_impl(Stream& x, short d); + static void pack_int_impl(Stream& x, int d); + static void pack_long_impl(Stream& x, long d); + static void pack_long_long_impl(Stream& x, long long d); + static void pack_unsigned_short_impl(Stream& x, unsigned short d); + static void pack_unsigned_int_impl(Stream& x, unsigned int d); + static void pack_unsigned_long_impl(Stream& x, unsigned long d); + static void pack_unsigned_long_long_impl(Stream& x, unsigned long long d); + static void pack_float_impl(Stream& x, float d); static void pack_double_impl(Stream& x, double d); + static void pack_nil_impl(Stream& x); static void pack_true_impl(Stream& x); static void pack_false_impl(Stream& x); + static void pack_array_impl(Stream& x, unsigned int n); + static void pack_map_impl(Stream& x, unsigned int n); + static void pack_raw_impl(Stream& x, size_t l); static void pack_raw_body_impl(Stream& x, const void* b, size_t l); @@ -88,8 +109,15 @@ private: #define msgpack_pack_inline_func(name) \ template \ inline void packer::pack_ ## name ## _impl + +#define msgpack_pack_inline_func_cint(name) \ + template \ + inline void packer::pack_ ## name ## _impl + #define msgpack_pack_user Stream& + #define msgpack_pack_append_buffer append_buffer + #include "msgpack/pack_template.h" diff --git a/cpp/type/integer.hpp b/cpp/type/integer.hpp index 3fd57b4..5cd10f6 100644 --- a/cpp/type/integer.hpp +++ b/cpp/type/integer.hpp @@ -63,102 +63,6 @@ namespace detail { return detail::convert_integer_sign::is_signed>::convert(o); } - - - template - struct pack_integer_size_sign; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) - { o.pack_int8(v); } - }; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) - { o.pack_uint8(v); } - }; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) { - if( (int16_t)v <= (int16_t)std::numeric_limits::max() && - (int16_t)v >= (int16_t)std::numeric_limits::min()) - { o.pack_int8(v); } - else { o.pack_int16(v); } - } - }; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) { - if( (uint16_t)v <= (uint16_t)std::numeric_limits::max()) - { o.pack_uint8(v); } - else { o.pack_uint16(v); } - } - }; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) { - if( (int32_t)v <= (int32_t)std::numeric_limits::max() && - (int32_t)v >= (int32_t)std::numeric_limits::min()) - { o.pack_int8(v); } - else if( (int32_t)v <= (int32_t)std::numeric_limits::max() && - (int32_t)v >= (int32_t)std::numeric_limits::min()) - { o.pack_int16(v); } - else { o.pack_int32(v); } - } - }; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) { - if( (uint32_t)v <= (uint32_t)std::numeric_limits::max()) - { o.pack_uint8(v); } - else if( (uint32_t)v <= (uint32_t)std::numeric_limits::max()) - { o.pack_uint16(v); } - else { o.pack_uint32(v); } - } - }; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) { - if( (int64_t)v <= (int64_t)std::numeric_limits::max() && - (int64_t)v >= (int64_t)std::numeric_limits::min()) - { o.pack_int8(v); } - else if( (int64_t)v <= (int64_t)std::numeric_limits::max() && - (int64_t)v >= (int64_t)std::numeric_limits::min()) - { o.pack_int16(v); } - else if( (int64_t)v <= (int64_t)std::numeric_limits::max() && - (int64_t)v >= (int64_t)std::numeric_limits::min()) - { o.pack_int32(v); } - else { o.pack_int64(v); } - } - }; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) { - if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) - { o.pack_uint8(v); } - else if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) - { o.pack_uint16(v); } - else if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) - { o.pack_uint32(v); } - else { o.pack_uint64(v); } - } - }; - - - template - static inline void pack_integer(T v, packer& o) - { - pack_integer_size_sign::is_signed>::pack(o, v); - } - } // namespace detail } // namespace type @@ -197,44 +101,44 @@ inline unsigned long long& operator>> (object o, unsigned long long& v) template inline packer& operator<< (packer& o, const signed char& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_int8(v); return o; } template inline packer& operator<< (packer& o, const signed short& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_short(v); return o; } template inline packer& operator<< (packer& o, const signed int& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_int(v); return o; } template inline packer& operator<< (packer& o, const signed long& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_long(v); return o; } template inline packer& operator<< (packer& o, const signed long long& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_long_long(v); return o; } template inline packer& operator<< (packer& o, const unsigned char& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_uint8(v); return o; } template inline packer& operator<< (packer& o, const unsigned short& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_unsigned_short(v); return o; } template inline packer& operator<< (packer& o, const unsigned int& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_unsigned_int(v); return o; } template inline packer& operator<< (packer& o, const unsigned long& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_unsigned_long(v); return o; } template inline packer& operator<< (packer& o, const unsigned long long& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_unsigned_long_long(v); return o; } } // namespace msgpack diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h new file mode 100644 index 0000000..6512d77 --- /dev/null +++ b/msgpack/pack_define.h @@ -0,0 +1,26 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_DEFINE_H__ +#define MSGPACK_PACK_DEFINE_H__ + +#include +#include +#include + +#endif /* msgpack/pack_define.h */ + diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 94fbfd4..dafb0b1 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -50,7 +50,6 @@ #endif - #ifndef msgpack_pack_inline_func #error msgpack_pack_inline_func template is not defined #endif @@ -68,7 +67,127 @@ * Integer */ -#define msgpack_pack_compress_int32(x, d) \ +#define msgpack_pack_real_uint8(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ +} while(0) + +#define msgpack_pack_real_uint16(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } else if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ +} while(0) + +#define msgpack_pack_real_uint32(x, d) \ +do { \ + if(d < (1<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_uint64(x, d) \ +do { \ + if(d < (1ULL<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1ULL<<16)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else if(d < (1ULL<<32)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int8(x, d) \ +do { \ + if(d < -(1<<5)) { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } \ +} while(0) + +#define msgpack_pack_real_int16(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int32(x, d) \ do { \ if(d < -(1<<5)) { \ if(d < -(1<<15)) { \ @@ -104,31 +223,7 @@ do { \ } \ } while(0) -#define msgpack_pack_compress_uint32(x, d) \ -do { \ - if(d < (1<<8)) { \ - if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); \ - } else { \ - /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ - } else { \ - if(d < (1<<16)) { \ - /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else { \ - /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ - msgpack_pack_append_buffer(x, buf, 5); \ - } \ - } \ -} while(0) - -#define msgpack_pack_compress_int64(x, d) \ +#define msgpack_pack_real_int64(x, d) \ do { \ if(d < -(1LL<<5)) { \ if(d < -(1LL<<15)) { \ @@ -180,133 +275,276 @@ do { \ } \ } while(0) -#define msgpack_pack_compress_uint64(x, d) \ -do { \ - if(d < (1ULL<<8)) { \ - if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); \ - } else { \ - /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ - } else { \ - if(d < (1ULL<<16)) { \ - /* signed 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else if(d < (1ULL<<32)) { \ - /* signed 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ - msgpack_pack_append_buffer(x, buf, 5); \ - } else { \ - /* signed 64 */ \ - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ - msgpack_pack_append_buffer(x, buf, 9); \ - } \ - } \ -} while(0) -msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) +#ifdef msgpack_pack_inline_func_fastint + +msgpack_pack_inline_func_fastint(uint8)(msgpack_pack_user x, uint8_t d) { -#if SIZEOF_INT == 8 - msgpack_pack_compress_int64(x, d); -#else - msgpack_pack_compress_int32(x, d); -#endif + const unsigned char buf[2] = {0xcc, d}; + msgpack_pack_append_buffer(x, buf, 2); } -msgpack_pack_inline_func(unsigned_int)(msgpack_pack_user x, unsigned int d) -{ -#if SIZEOF_INT == 8 - msgpack_pack_compress_uint64(x, d); -#else - msgpack_pack_compress_uint32(x, d); -#endif -} - -msgpack_pack_inline_func(long)(msgpack_pack_user x, long d) -{ -#if SIZEOF_LONG == 8 - msgpack_pack_compress_int64(x, d); -#else - msgpack_pack_compress_int32(x, d); -#endif -} - -msgpack_pack_inline_func(unsigned_long)(msgpack_pack_user x, unsigned long d) -{ -#if SIZEOF_LONG == 8 - msgpack_pack_compress_uint64(x, d); -#else - msgpack_pack_compress_uint32(x, d); -#endif -} - -#undef msgpack_pack_compress_int32 -#undef msgpack_pack_compress_uint32 -#undef msgpack_pack_compress_int64 -#undef msgpack_pack_compress_uint64 - -msgpack_pack_inline_func(uint8)(msgpack_pack_user x, uint8_t d) -{ - if(d < 128) { - msgpack_pack_append_buffer(x, &d, 1); - } else { - const unsigned char buf[2] = {0xcc, d}; - msgpack_pack_append_buffer(x, buf, 2); - } -} - -msgpack_pack_inline_func(uint16)(msgpack_pack_user x, uint16_t d) +msgpack_pack_inline_func_fastint(uint16)(msgpack_pack_user x, uint16_t d) { const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func(uint32)(msgpack_pack_user x, uint32_t d) +msgpack_pack_inline_func_fastint(uint32)(msgpack_pack_user x, uint32_t d) { const unsigned char buf[5] = {0xce, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func(uint64)(msgpack_pack_user x, uint64_t d) +msgpack_pack_inline_func_fastint(uint64)(msgpack_pack_user x, uint64_t d) { const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } -msgpack_pack_inline_func(int8)(msgpack_pack_user x, int8_t d) +msgpack_pack_inline_func_fastint(int8)(msgpack_pack_user x, int8_t d) { - if(d > 0) { - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); - } else if(d >= -32) { - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); - } else { - const unsigned char buf[2] = {0xd0, d}; - msgpack_pack_append_buffer(x, buf, 2); - } + const unsigned char buf[2] = {0xd0, d}; + msgpack_pack_append_buffer(x, buf, 2); } -msgpack_pack_inline_func(int16)(msgpack_pack_user x, int16_t d) +msgpack_pack_inline_func_fastint(int16)(msgpack_pack_user x, int16_t d) { const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func(int32)(msgpack_pack_user x, int32_t d) +msgpack_pack_inline_func_fastint(int32)(msgpack_pack_user x, int32_t d) { const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func(int64)(msgpack_pack_user x, int64_t d) +msgpack_pack_inline_func_fastint(int64)(msgpack_pack_user x, int64_t d) { const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } +#undef msgpack_pack_inline_func_fastint +#endif + + +msgpack_pack_inline_func(uint8)(msgpack_pack_user x, uint8_t d) +{ + msgpack_pack_real_uint8(x, d); +} + +msgpack_pack_inline_func(uint16)(msgpack_pack_user x, uint16_t d) +{ + msgpack_pack_real_uint16(x, d); +} + +msgpack_pack_inline_func(uint32)(msgpack_pack_user x, uint32_t d) +{ + msgpack_pack_real_uint32(x, d); +} + +msgpack_pack_inline_func(uint64)(msgpack_pack_user x, uint64_t d) +{ + msgpack_pack_real_uint64(x, d); +} + +msgpack_pack_inline_func(int8)(msgpack_pack_user x, int8_t d) +{ + msgpack_pack_real_int8(x, d); +} + +msgpack_pack_inline_func(int16)(msgpack_pack_user x, int16_t d) +{ + msgpack_pack_real_int16(x, d); +} + +msgpack_pack_inline_func(int32)(msgpack_pack_user x, int32_t d) +{ + msgpack_pack_real_int32(x, d); +} + +msgpack_pack_inline_func(int64)(msgpack_pack_user x, int64_t d) +{ + msgpack_pack_real_int64(x, d); +} + + +#ifdef msgpack_pack_inline_func_cint + +msgpack_pack_inline_func_cint(short)(msgpack_pack_user x, short d) +{ +#if defined(SIZEOF_SHORT) || defined(SHRT_MAX) +#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(short) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(short) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(int)(msgpack_pack_user x, int d) +{ +#if defined(SIZEOF_INT) || defined(INT_MAX) +#if SIZEOF_INT == 2 || INT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(int) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(int) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(long)(msgpack_pack_user x, long d) +{ +#if defined(SIZEOF_LONG) || defined(LONG_MAX) +#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(long_long)(msgpack_pack_user x, long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(unsigned_short)(msgpack_pack_user x, unsigned short d) +{ +#if defined(SIZEOF_SHORT) || defined(USHRT_MAX) +#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned short) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned short) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(unsigned_int)(msgpack_pack_user x, unsigned int d) +{ +#if defined(SIZEOF_INT) || defined(UINT_MAX) +#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(unsigned_long)(msgpack_pack_user x, unsigned long d) +{ +#if defined(SIZEOF_LONG) || defined(ULONG_MAX) +#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(unsigned_long_long)(msgpack_pack_user x, unsigned long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned long long) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned long long) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +#undef msgpack_pack_inline_func_cint +#endif + + /* * Float @@ -430,3 +668,12 @@ msgpack_pack_inline_func(raw_body)(msgpack_pack_user x, const void* b, size_t l) #undef STORE_BE32 #undef STORE_BE64 +#undef msgpack_pack_real_uint8 +#undef msgpack_pack_real_uint16 +#undef msgpack_pack_real_uint32 +#undef msgpack_pack_real_uint64 +#undef msgpack_pack_real_int8 +#undef msgpack_pack_real_int16 +#undef msgpack_pack_real_int32 +#undef msgpack_pack_real_int64 + diff --git a/ruby/gem/Manifest.txt b/ruby/gem/Manifest.txt index 388135e..17e2b78 100644 --- a/ruby/gem/Manifest.txt +++ b/ruby/gem/Manifest.txt @@ -10,6 +10,7 @@ ext/pack.h ext/rbinit.c ext/unpack.c ext/unpack.h +msgpack/pack_define.h msgpack/pack_template.h msgpack/unpack_define.h msgpack/unpack_template.h diff --git a/ruby/gengem.sh b/ruby/gengem.sh index 000c708..de9f4a5 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -7,6 +7,7 @@ cp rbinit.c gem/ext/ cp unpack.c gem/ext/ cp unpack.h gem/ext/ #cp ../README gem/README.txt +cp ../msgpack/pack_define.h gem/msgpack/ cp ../msgpack/pack_template.h gem/msgpack/ cp ../msgpack/unpack_define.h gem/msgpack/ cp ../msgpack/unpack_template.h gem/msgpack/ diff --git a/ruby/pack.c b/ruby/pack.c index 5262024..c399c25 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -16,41 +16,19 @@ * limitations under the License. */ #include "ruby.h" -#include -#include +#include "msgpack/pack_define.h" #define msgpack_pack_inline_func(name) \ static inline void msgpack_pack_##name +#define msgpack_pack_inline_func_cint(name) \ + static inline void msgpack_pack_##name + #define msgpack_pack_user VALUE #define msgpack_pack_append_buffer(user, buf, len) \ rb_str_buf_cat(user, (const void*)buf, len) -/* -static void msgpack_pack_int(VALUE x, int d); -static void msgpack_pack_unsigned_int(VALUE x, unsigned int d); -static void msgpack_pack_long(VALUE x, long d); -static void msgpack_pack_unsigned_long(VALUE x, unsigned long d); -static void msgpack_pack_uint8(VALUE x, uint8_t d); -static void msgpack_pack_uint16(VALUE x, uint16_t d); -static void msgpack_pack_uint32(VALUE x, uint32_t d); -static void msgpack_pack_uint64(VALUE x, uint64_t d); -static void msgpack_pack_int8(VALUE x, int8_t d); -static void msgpack_pack_int16(VALUE x, int16_t d); -static void msgpack_pack_int32(VALUE x, int32_t d); -static void msgpack_pack_int64(VALUE x, int64_t d); -static void msgpack_pack_float(VALUE x, float d); -static void msgpack_pack_double(VALUE x, double d); -static void msgpack_pack_nil(VALUE x); -static void msgpack_pack_true(VALUE x); -static void msgpack_pack_false(VALUE x); -static void msgpack_pack_array(VALUE x, unsigned int n); -static void msgpack_pack_map(VALUE x, unsigned int n); -static void msgpack_pack_raw(VALUE x, size_t l); -static void msgpack_pack_raw_body(VALUE x, const void* b, size_t l); -*/ - #include "msgpack/pack_template.h" From c93d45371b2b2a085fa2fe84d6ceb79decf778d2 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0044/1172] c++ binding: fix missing packer::pack_unsigned_long git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@91 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/pack.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/pack.hpp b/cpp/pack.hpp index e860f4c..c07bcc3 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -47,6 +47,7 @@ public: void pack_long_long(long long d) { pack_long_long_impl(m_stream, d); } void pack_unsigned_short(unsigned short d) { pack_unsigned_short_impl(m_stream, d); } void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } + void pack_unsigned_long(unsigned long d) { pack_unsigned_long_impl(m_stream, d); } void pack_unsigned_long_long(unsigned long long d) { pack_unsigned_long_long_impl(m_stream, d); } void pack_float(float d) { pack_float_impl(m_stream, d); } From e582fa34c7f1246d27cce5a2333e0f954cbed251 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0045/1172] c++ binding: improve msgpack::zone, zero-copy stream deserializer git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@92 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/Makefile.am | 13 ++- cpp/unpack.cpp | 233 +++++++++++++++++++++++++++++++---------------- cpp/unpack.hpp | 8 +- cpp/zone.cpp | 82 ++++++++++++++--- cpp/zone.hpp | 86 ----------------- cpp/zone.hpp.erb | 103 +++++++++++++++++++++ 6 files changed, 336 insertions(+), 189 deletions(-) delete mode 100644 cpp/zone.hpp create mode 100644 cpp/zone.hpp.erb diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 19c6007..45c75e0 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -26,14 +26,21 @@ noinst_HEADERS = \ # FIXME object.lo: msgpack/type/tuple.hpp -unpack.lo: msgpack/type/tuple.hpp -zone.lo: msgpack/type/tuple.hpp +unpack.lo: msgpack/type/tuple.hpp msgpack/zone.hpp +zone.lo: msgpack/type/tuple.hpp msgpack/zone.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb $(ERB) $< > $@.tmp mv $@.tmp $@ -MOSTLYCLEANFILES = msgpack/type/tuple.hpp +msgpack/zone.hpp: msgpack/zone.hpp.erb + $(ERB) $< > $@.tmp + mv $@.tmp $@ +MOSTLYCLEANFILES = \ + msgpack/type/tuple.hpp \ + msgpack/zone.hpp + +# FIXME libmsgpack_la_LDFLAGS = -version-info 0:0:0 diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 8edf407..f2ef330 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -22,6 +22,19 @@ namespace msgpack { +//namespace { +struct allocator { + zone* z; + bool referenced; + + inline object* malloc_object(size_t n) + { + return (object*)z->malloc(sizeof(object)*n); + } +}; +//} // noname namespace + + #define msgpack_unpack_struct(name) \ struct msgpack_unpacker_##name @@ -33,7 +46,7 @@ namespace msgpack { #define msgpack_unpack_object object -#define msgpack_unpack_user zone* +#define msgpack_unpack_user allocator struct msgpack_unpacker_context; @@ -46,92 +59,100 @@ static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, const char* data, size_t len, size_t* off); -static inline object msgpack_unpack_init(zone** z) +static inline object msgpack_unpack_init(allocator* a) { return object(); } -static inline object msgpack_unpack_uint8(zone** z, uint8_t d) +static inline object msgpack_unpack_uint8(allocator* a, uint8_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_uint16(zone** z, uint16_t d) +static inline object msgpack_unpack_uint16(allocator* a, uint16_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_uint32(zone** z, uint32_t d) +static inline object msgpack_unpack_uint32(allocator* a, uint32_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_uint64(zone** z, uint64_t d) +static inline object msgpack_unpack_uint64(allocator* a, uint64_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_int8(zone** z, int8_t d) +static inline object msgpack_unpack_int8(allocator* a, int8_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_int16(zone** z, int16_t d) +static inline object msgpack_unpack_int16(allocator* a, int16_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_int32(zone** z, int32_t d) +static inline object msgpack_unpack_int32(allocator* a, int32_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_int64(zone** z, int64_t d) +static inline object msgpack_unpack_int64(allocator* a, int64_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_float(zone** z, float d) +static inline object msgpack_unpack_float(allocator* a, float d) { object o; o.type = type::DOUBLE; o.via.dec = d; return o; } -static inline object msgpack_unpack_double(zone** z, double d) +static inline object msgpack_unpack_double(allocator* a, double d) { object o; o.type = type::DOUBLE; o.via.dec = d; return o; } -static inline object msgpack_unpack_nil(zone** z) +static inline object msgpack_unpack_nil(allocator* a) { object o; o.type = type::NIL; return o; } -static inline object msgpack_unpack_true(zone** z) +static inline object msgpack_unpack_true(allocator* a) { object o; o.type = type::BOOLEAN; o.via.boolean = true; return o; } -static inline object msgpack_unpack_false(zone** z) +static inline object msgpack_unpack_false(allocator* a) { object o; o.type = type::BOOLEAN; o.via.boolean = false; return o; } -static inline object msgpack_unpack_array(zone** z, unsigned int n) +static inline object msgpack_unpack_array(allocator* a, unsigned int n) { object o; o.type = type::ARRAY; o.via.container.size = 0; - o.via.container.ptr = (*z)->malloc_container(n); + o.via.container.ptr = a->malloc_object(n); return o; } -static inline void msgpack_unpack_array_item(zone** z, object* c, object o) +static inline void msgpack_unpack_array_item(allocator* a, object* c, object o) { c->via.container.ptr[ c->via.container.size++ ] = o; } -static inline object msgpack_unpack_map(zone** z, unsigned int n) +static inline object msgpack_unpack_map(allocator* a, unsigned int n) { object o; o.type = type::MAP; o.via.container.size = 0; - o.via.container.ptr = (*z)->malloc_container(n*2); + o.via.container.ptr = a->malloc_object(n*2); return o; } -static inline void msgpack_unpack_map_item(zone** z, object* c, object k, object v) +static inline void msgpack_unpack_map_item(allocator* a, object* c, object k, object v) { c->via.container.ptr[ c->via.container.size ] = k; c->via.container.ptr[ c->via.container.size+1 ] = v; ++c->via.container.size; } -static inline object msgpack_unpack_raw(zone** z, const char* b, const char* p, unsigned int l) -{ object o; o.type = type::RAW; o.via.ref.ptr = p; o.via.ref.size = l; return o; } - +static inline object msgpack_unpack_raw(allocator* a, const char* b, const char* p, unsigned int l) +{ + object o; + o.type = type::RAW; + o.via.ref.ptr = p; + o.via.ref.size = l; + a->referenced = true; + return o; +} #include "msgpack/unpack_template.h" -struct unpacker::context { - context(zone* z) +namespace { +struct context { + context() { msgpack_unpacker_init(&m_ctx); - m_ctx.user = z; + allocator a = {NULL, false}; + m_ctx.user = a; } ~context() { } @@ -148,35 +169,64 @@ struct unpacker::context { void reset() { - zone* z = m_ctx.user; + zone* z = m_ctx.user.z; msgpack_unpacker_init(&m_ctx); - m_ctx.user = z; + allocator a = {z, false}; + m_ctx.user = a; } - void reset(zone* z) + void set_zone(zone* z) { - msgpack_unpacker_init(&m_ctx); - m_ctx.user = z; + m_ctx.user.z = z; } - zone* user() + bool is_referenced() const { - return m_ctx.user; - } - - void user(zone* z) - { - m_ctx.user = z; + return m_ctx.user.referenced; } private: msgpack_unpacker_context m_ctx; + zone* m_zone; private: - context(); context(const context&); }; +context* as_ctx(void* m) +{ + return reinterpret_cast(m); +} + + +static const size_t COUNTER_SIZE = sizeof(unsigned int); + +static inline void init_count(void* buffer) +{ + *(volatile unsigned int*)buffer = 1; +} + +static inline void decl_count(void* buffer) +{ + //if(--*(unsigned int*)buffer == 0) { + if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { + free(buffer); + } +} + +static inline void incr_count(void* buffer) +{ + //++*(unsigned int*)buffer; + __sync_add_and_fetch((unsigned int*)buffer, 1); +} + +static inline unsigned int get_count(void* buffer) +{ + return *(volatile unsigned int*)buffer; +} + +} // noname namespace + unpacker::unpacker(size_t initial_buffer_size) : m_buffer(NULL), @@ -184,45 +234,77 @@ unpacker::unpacker(size_t initial_buffer_size) : m_free(0), m_off(0), m_zone(new zone()), - m_ctx(new context(&*m_zone)), + m_ctx(new context()), m_initial_buffer_size(initial_buffer_size) -{ } +{ + if(m_initial_buffer_size < COUNTER_SIZE) { + m_initial_buffer_size = COUNTER_SIZE; + } + + as_ctx(m_ctx)->set_zone(m_zone.get()); + + m_buffer = (char*)::malloc(m_initial_buffer_size); + if(!m_buffer) { throw std::bad_alloc(); } + init_count(m_buffer); + + m_used = COUNTER_SIZE; + m_free = m_initial_buffer_size - m_used; + m_off = COUNTER_SIZE; +} unpacker::~unpacker() { - delete m_ctx; + delete as_ctx(m_ctx); + decl_count(m_buffer); } - void unpacker::expand_buffer(size_t len) { - if(m_off == 0) { - size_t next_size; - if(m_used != 0) { next_size = (m_used + m_free) * 2; } - else { next_size = m_initial_buffer_size; } + if(m_used == m_off && get_count(m_buffer) == 1 && + !as_ctx(m_ctx)->is_referenced()) { + // rewind buffer + m_free += m_used - COUNTER_SIZE; + m_used = COUNTER_SIZE; + m_off = COUNTER_SIZE; + if(m_free >= len) { return; } + } + + if(m_off == COUNTER_SIZE) { + size_t next_size = (m_used + m_free) * 2; while(next_size < len + m_used) { next_size *= 2; } - m_buffer = m_zone->realloc(m_buffer, next_size); + char* tmp = (char*)::realloc(m_buffer, next_size); + if(!tmp) { throw std::bad_alloc(); } + + m_buffer = tmp; m_free = next_size - m_used; } else { - size_t next_size = m_initial_buffer_size; - while(next_size < len + m_used - m_off) { next_size *= 2; } + size_t next_size = m_initial_buffer_size; // include COUNTER_SIZE + size_t not_parsed = m_used - m_off; + while(next_size < len + not_parsed + COUNTER_SIZE) { next_size *= 2; } - char* tmp = m_zone->malloc(next_size); - memcpy(tmp, m_buffer+m_off, m_used-m_off); + char* tmp = (char*)::malloc(next_size); + if(!tmp) { throw std::bad_alloc(); } + init_count(tmp); + + try { + m_zone->push_finalizer(decl_count, m_buffer); + } catch (...) { free(tmp); throw; } + + memcpy(tmp+COUNTER_SIZE, m_buffer+m_off, not_parsed); m_buffer = tmp; - m_used = m_used - m_off; + m_used = not_parsed + COUNTER_SIZE; m_free = next_size - m_used; - m_off = 0; + m_off = COUNTER_SIZE; } } bool unpacker::execute() { - int ret = m_ctx->execute(m_buffer, m_used, &m_off); + int ret = as_ctx(m_ctx)->execute(m_buffer, m_used, &m_off); if(ret < 0) { throw unpack_error("parse error"); } else if(ret == 0) { @@ -234,56 +316,45 @@ bool unpacker::execute() zone* unpacker::release_zone() { + m_zone->push_finalizer(decl_count, m_buffer); + incr_count(m_buffer); + + //std::auto_ptr old(new zone()); + //m_zone.swap(old); zone* n = new zone(); std::auto_ptr old(m_zone.release()); m_zone.reset(n); - //std::auto_ptr old(new zone()); - //m_zone.swap(old); + as_ctx(m_ctx)->set_zone(m_zone.get()); - // move all bytes in m_buffer to new buffer from the new zone - if(m_used <= m_off) { - m_buffer = NULL; - m_used = 0; - m_free = 0; - m_off = 0; - } else { - try { - expand_buffer(0); - } catch (...) { - // m_zone.swap(old); - zone* tmp = old.release(); - old.reset(m_zone.release()); - m_zone.reset(tmp); - throw; - } - } - m_ctx->user(m_zone.get()); return old.release(); } object unpacker::data() { - return m_ctx->data(); + return as_ctx(m_ctx)->data(); } void unpacker::reset() { - if(m_off != 0) { delete release_zone(); } - m_ctx->reset(); + if(!m_zone->empty()) { delete release_zone(); } + as_ctx(m_ctx)->reset(); } object unpacker::unpack(const char* data, size_t len, zone& z, size_t* off) { - context ctx(&z); + context ctx; + ctx.set_zone(&z); if(off) { - int ret = ctx.execute(data, len, off); + size_t noff = *off; + int ret = ctx.execute(data, len, &noff); if(ret < 0) { throw unpack_error("parse error"); } else if(ret == 0) { throw unpack_error("insufficient bytes"); } + *off = noff; } else { size_t noff = 0; int ret = ctx.execute(data, len, &noff); diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index ac20de3..cba1963 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -24,7 +24,8 @@ #include #ifndef MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE -#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE 8*1024 +#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE 16 +//#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE 8*1024 #endif namespace msgpack { @@ -133,10 +134,9 @@ private: std::auto_ptr m_zone; - struct context; - context* m_ctx; + void* m_ctx; - const size_t m_initial_buffer_size; + size_t m_initial_buffer_size; private: void expand_buffer(size_t len); diff --git a/cpp/zone.cpp b/cpp/zone.cpp index 527cc9c..7c5a1c9 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -16,31 +16,83 @@ // limitations under the License. // #include "msgpack/zone.hpp" +#include namespace msgpack { -void zone::clear() +zone::zone(size_t chunk_size) : + m_chunk_size(chunk_size) { - for(std::vector::iterator it(m_ptrs.begin()), it_end(m_ptrs.end()); - it != it_end; ++it) { - free(*it); - } - m_ptrs.clear(); + chunk dummy = {0, NULL, NULL}; + m_chunk_array.push_back(dummy); } -char* zone::realloc_real(char* ptr, size_t count) +zone::~zone() { - for(std::vector::reverse_iterator it(m_ptrs.rbegin()), it_end(m_ptrs.rend()); - it != it_end; ++it) { - if(*it == ptr) { - char* tmp = (char*)::realloc(ptr, count); - if(!tmp) { throw std::bad_alloc(); } - *it = tmp; - return tmp; + clear(); +} + +namespace { + template + struct zone_finalize { + void operator() (Private& f) { + (*f.func)(f.obj); } + }; + + template + struct zone_free { + void operator() (Private& c) { + ::free(c.alloc); + } + }; +} + +void zone::clear() +{ + std::for_each(m_finalizers.rbegin(), m_finalizers.rend(), + zone_finalize()); + m_finalizers.clear(); + + std::for_each(m_chunk_array.begin(), m_chunk_array.end(), + zone_free()); + m_chunk_array.resize(1); + m_chunk_array[0].ptr = NULL; + m_chunk_array[0].free = 0; +} + +bool zone::empty() const +{ + return m_chunk_array.back().alloc == NULL && + m_finalizers.empty(); +} + +void* zone::malloc(size_t size) +{ + if(m_chunk_array.back().free > size) { + char* p = (char*)m_chunk_array.back().ptr; + m_chunk_array.back().ptr = p + size; + m_chunk_array.back().free -= size; + return p; } - throw std::bad_alloc(); + + size_t sz = m_chunk_size; + while(sz < size) { sz *= 2; } + + chunk dummy = {0, NULL, NULL}; + m_chunk_array.push_back(dummy); + + char* p = (char*)::malloc(sz); + if(!p) { + m_chunk_array.pop_back(); + throw std::bad_alloc(); + } + + m_chunk_array.back().free = sz - size; + m_chunk_array.back().ptr = p + size; + m_chunk_array.back().alloc = p; + return p; } diff --git a/cpp/zone.hpp b/cpp/zone.hpp deleted file mode 100644 index e7e73e1..0000000 --- a/cpp/zone.hpp +++ /dev/null @@ -1,86 +0,0 @@ -// -// MessagePack for C++ memory pool -// -// Copyright (C) 2008 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#ifndef MSGPACK_ZONE_HPP__ -#define MSGPACK_ZONE_HPP__ - -#include "msgpack/object.hpp" -#include -#include - -namespace msgpack { - - -class zone { -public: - zone(); - ~zone(); - -public: - char* malloc(size_t count); - char* realloc(char* ptr, size_t count); - object* malloc_container(size_t count); - - void clear(); - -private: - std::vector m_ptrs; - -private: - char* realloc_real(char* ptr, size_t count); - -private: - zone(const zone&); -}; - - -inline zone::zone() { } - -inline zone::~zone() { clear(); } - -inline char* zone::malloc(size_t count) -{ - char* ptr = (char*)::malloc(count); - if(!ptr) { throw std::bad_alloc(); } - try { - m_ptrs.push_back(ptr); - } catch (...) { - free(ptr); - throw; - } - return ptr; -} - -inline char* zone::realloc(char* ptr, size_t count) -{ - if(ptr == NULL) { - return zone::malloc(count); - } else { - return realloc_real(ptr, count); - } -} - -inline object* zone::malloc_container(size_t count) -{ - return (object*)zone::malloc(sizeof(object)*count); -} - - -} // namespace msgpack - -#endif /* msgpack/zone.hpp */ - diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb new file mode 100644 index 0000000..9a8c81a --- /dev/null +++ b/cpp/zone.hpp.erb @@ -0,0 +1,103 @@ +// +// MessagePack for C++ memory pool +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_ZONE_HPP__ +#define MSGPACK_ZONE_HPP__ + +#include "msgpack/object.hpp" +#include +#include + +#ifndef MSGPACK_ZONE_CHUNK_SIZE +#define MSGPACK_ZONE_CHUNK_SIZE 2048 +#endif +<% GENERATION_LIMIT = 15 %> +namespace msgpack { + + +class zone { +public: + zone(size_t chunk_size = MSGPACK_ZONE_CHUNK_SIZE); + ~zone(); + +public: + void* malloc(size_t size); + + void push_finalizer(void (*func)(void*), void* obj); + + void clear(); + + bool empty() const; + + <%0.upto(GENERATION_LIMIT) {|i|%> + template , typename A<%=j%><%}%>> + T* allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>); + <%}%> + +private: + struct chunk { + size_t free; + void* ptr; + void* alloc; + }; + + std::vector m_chunk_array; + + struct finalizer { + void (*func)(void*); + void* obj; + }; + + std::vector m_finalizers; + + template + static void object_destructor(void* obj); + + size_t m_chunk_size; + +private: + zone(const zone&); +}; + + +inline void zone::push_finalizer(void (*func)(void*), void* obj) +{ + finalizer f = {func, obj}; + m_finalizers.push_back(f); +} + +template +void zone::object_destructor(void* obj) +{ + reinterpret_cast(obj)->~T(); +} + +<%0.upto(GENERATION_LIMIT) {|i|%> +template , typename A<%=j%><%}%>> +T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) +{ + void* x = malloc(sizeof(T)); + push_finalizer(&zone::object_destructor, x); + try { return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); } + catch (...) { m_finalizers.pop_back(); throw; } +} +<%}%> + +} // namespace msgpack + +#endif /* msgpack/zone.hpp */ + From cbf2be8db4f6bc18e1ec3002534da06774561a4d Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:01 +0000 Subject: [PATCH 0046/1172] c++ binding: remove unpacker::parsed_size() API git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@93 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/unpack.hpp | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index cba1963..fe20fc7 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -109,23 +109,19 @@ public: public: // These functions are usable when non-MessagePack message follows after // MessagePack message. - // Note that there are no parsed buffer when execute() returned true. - /*! get address of buffer that is not parsed */ + /*! get address of the buffer that is not parsed */ char* nonparsed_buffer(); size_t nonparsed_size() const; - /*! get the number of bytes that is already parsed */ - size_t parsed_size() const; + /*! skip specified size of non-parsed buffer, leaving the buffer */ + // Note that the `len' argument must be smaller than nonparsed_size() + void skip_nonparsed_buffer(size_t len); /*! remove unparsed buffer from unpacker */ // Note that reset() leaves non-parsed buffer. void remove_nonparsed_buffer(); - /*! skip specified size of non-parsed buffer, leaving the buffer */ - // Note the size must be smaller than nonparsed_size() - void skip_nonparsed_buffer(size_t len); - private: char* m_buffer; size_t m_used; @@ -174,15 +170,12 @@ inline char* unpacker::nonparsed_buffer() inline size_t unpacker::nonparsed_size() const { return m_used - m_off; } -inline size_t unpacker::parsed_size() const - { return m_off; } +inline void unpacker::skip_nonparsed_buffer(size_t len) + { m_off += len; } inline void unpacker::remove_nonparsed_buffer() { m_used = m_off; } -inline void unpacker::skip_nonparsed_buffer(size_t len) - { m_off += len; } - inline object unpack(const char* data, size_t len, zone& z, size_t* off = NULL) { From 823add403e5f0d905fc0003023c2b270adb22897 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:01 +0000 Subject: [PATCH 0047/1172] merge 0.2.2 git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@94 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- configure.in | 2 +- msgpack/unpack_define.h | 4 +++- ruby/gem/lib/msgpack/version.rb | 2 +- ruby/msgpack.gemspec | 11 +++++++++++ ruby/unpack.c | 13 +++++++++++-- 5 files changed, 27 insertions(+), 5 deletions(-) create mode 100755 ruby/msgpack.gemspec diff --git a/configure.in b/configure.in index 63762b0..3a67081 100644 --- a/configure.in +++ b/configure.in @@ -1,5 +1,5 @@ AC_INIT(msgpack/unpack_template.h) -AM_INIT_AUTOMAKE(msgpack, 0.2.1) +AM_INIT_AUTOMAKE(msgpack, 0.2.2) AC_CONFIG_HEADER(config.h) AC_PROG_LIBTOOL diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 1d9db19..f639841 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -22,8 +22,10 @@ #include #include #include -#include #include +#ifndef __WIN32__ +#include +#endif #ifdef __cplusplus extern "C" { diff --git a/ruby/gem/lib/msgpack/version.rb b/ruby/gem/lib/msgpack/version.rb index 229c746..488e0b2 100644 --- a/ruby/gem/lib/msgpack/version.rb +++ b/ruby/gem/lib/msgpack/version.rb @@ -2,7 +2,7 @@ module MessagePack module VERSION #:nodoc: MAJOR = 0 MINOR = 2 - TINY = 1 + TINY = 2 STRING = [MAJOR, MINOR, TINY].join('.') end diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec new file mode 100755 index 0000000..db2fcd5 --- /dev/null +++ b/ruby/msgpack.gemspec @@ -0,0 +1,11 @@ +Gem::Specification.new do |s| + s.platform = Gem::Platform::CURRENT + s.name = "msgpack" + s.version = "0.2.2" + s.summary = "MessagePack" + s.author = "FURUHASHI Sadayuki" + s.email = "frsyuki _at_ users.sourceforge.jp" + s.homepage = "https://launchpad.net/msgpack/" + s.require_paths = ["lib", "ext"] + s.files = ["lib/**/*", "ext/**/*"].map {|g| Dir.glob(g) }.flatten +end diff --git a/ruby/unpack.c b/ruby/unpack.c index 8ab425c..df72246 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -21,6 +21,7 @@ typedef struct { int finished; + VALUE origstr; } msgpack_unpack_context; @@ -104,7 +105,7 @@ static inline void template_callback_map_item(msgpack_unpack_context* x, VALUE* { rb_hash_aset(*c, k, v); } static inline VALUE template_callback_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l) -{ return rb_str_new(p, l); } +{ return l == 0 ? rb_str_new(0,0) : rb_str_substr(x->origstr, p - b, l); } #include "msgpack/unpack_template.h" @@ -152,8 +153,9 @@ static VALUE MessagePack_Unpacker_alloc(VALUE klass) static VALUE MessagePack_Unpacker_reset(VALUE self) { UNPACKER(self, mp); - mp->user.finished = 0; msgpack_unpacker_init(mp); + msgpack_unpack_context ctx = {0, Qnil}; + mp->user = ctx; return self; } @@ -179,7 +181,9 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE args) rb_raise(eUnpackError, "offset is bigger than data buffer size."); } + mp->user.origstr = data; ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); + mp->user.origstr = Qnil; if(ret < 0) { rb_raise(eUnpackError, "parse error."); @@ -239,7 +243,9 @@ static VALUE MessagePack_unpack_impl(VALUE args) long dlen = RSTRING_LEN(data); int ret; + mp->user.origstr = data; ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); + mp->user.origstr = Qnil; if(ret < 0) { rb_raise(eUnpackError, "parse error."); @@ -268,6 +274,9 @@ static VALUE MessagePack_unpack(VALUE self, VALUE data) CHECK_STRING_TYPE(data); msgpack_unpacker mp; msgpack_unpacker_init(&mp); + msgpack_unpack_context ctx = {0, Qnil}; + mp.user = ctx; + rb_gc_disable(); VALUE args[2] = {(VALUE)&mp, data}; VALUE ret = rb_rescue(MessagePack_unpack_impl, (VALUE)args, From e893dde57e60e8e75b30b0e6a539ec233e112a4a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:01 +0000 Subject: [PATCH 0048/1172] C++ binding: efficient serializing interface git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@95 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.hpp | 11 ++++++----- cpp/test.cpp | 4 +++- cpp/unpack.hpp | 3 +-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 5f77d3e..1f0dcdf 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -135,14 +135,14 @@ inline void pack_copy(packer& o, T v) template inline T& operator>> (object o, T& v) { - v.msgpack_unpack(o); + v.msgpack_unpack(o.convert()); return v; } template inline packer& operator<< (packer& o, const T& v) { - o << v.msgpack_pack(); + v.msgpack_pack(o); return o; } @@ -156,14 +156,15 @@ public: define() {} define(msgpack_type v) : msgpack_type(v) {} - msgpack_type msgpack_pack() const + template + void msgpack_pack(Packer& o) const { - return *this; + o << static_cast(*this); } void msgpack_unpack(object o) { - convert(static_cast(*this), o); + o >> static_cast(*this); } }; diff --git a/cpp/test.cpp b/cpp/test.cpp index 02d461c..dd0b1fd 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -50,6 +50,7 @@ int main(void) { checker c; +#if 0 { // SimpleValue const char d[] = { 0x93, 0xc0, 0xc2, 0xc3, @@ -116,9 +117,10 @@ int main(void) ) ); } +#endif - static const unsigned TASK_ARRAY = 100; + static const unsigned TASK_ARRAY = 1000; static const unsigned TASK_REPEAT = 10; std::vector task; diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index fe20fc7..293a5d3 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -24,8 +24,7 @@ #include #ifndef MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE -#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE 16 -//#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE 8*1024 +#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE (32*1024) #endif namespace msgpack { From 2c7cdd5f40832b94c20ea578729ee00e3660d180 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:01 +0000 Subject: [PATCH 0049/1172] Ruby binding: add MessagePack::unpack_limit, MessagePack::Unpacker#execute_limit git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@96 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- configure.in | 2 +- ruby/gem/lib/msgpack/version.rb | 4 ++-- ruby/unpack.c | 30 ++++++++++++++++++++++-------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/configure.in b/configure.in index 3a67081..a464fb9 100644 --- a/configure.in +++ b/configure.in @@ -1,5 +1,5 @@ AC_INIT(msgpack/unpack_template.h) -AM_INIT_AUTOMAKE(msgpack, 0.2.2) +AM_INIT_AUTOMAKE(msgpack, 0.3.0) AC_CONFIG_HEADER(config.h) AC_PROG_LIBTOOL diff --git a/ruby/gem/lib/msgpack/version.rb b/ruby/gem/lib/msgpack/version.rb index 488e0b2..433e6fc 100644 --- a/ruby/gem/lib/msgpack/version.rb +++ b/ruby/gem/lib/msgpack/version.rb @@ -1,8 +1,8 @@ module MessagePack module VERSION #:nodoc: MAJOR = 0 - MINOR = 2 - TINY = 2 + MINOR = 3 + TINY = 0 STRING = [MAJOR, MINOR, TINY].join('.') end diff --git a/ruby/unpack.c b/ruby/unpack.c index df72246..2920240 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -169,12 +169,11 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE args) { VALUE self = ((VALUE*)args)[0]; VALUE data = ((VALUE*)args)[1]; - VALUE off = ((VALUE*)args)[2]; UNPACKER(self, mp); - size_t from = NUM2UINT(off); + size_t from = NUM2UINT(((VALUE*)args)[2]); char* dptr = RSTRING_PTR(data); - long dlen = RSTRING_LEN(data); + long dlen = FIX2LONG(((VALUE*)args)[3]); int ret; if(from >= dlen) { @@ -206,17 +205,24 @@ static VALUE MessagePack_Unpacker_execute_rescue(VALUE nouse) #endif } -static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) +static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, + VALUE off, VALUE limit) { // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ rb_gc_disable(); - VALUE args[3] = {self, data, off}; + VALUE args[4] = {self, data, off, limit}; VALUE ret = rb_rescue(MessagePack_Unpacker_execute_impl, (VALUE)args, MessagePack_Unpacker_execute_rescue, Qnil); rb_gc_enable(); return ret; } +static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) +{ + return MessagePack_Unpacker_execute_limit(self, data, off, + LONG2FIX(RSTRING_LEN(data))); +} + static VALUE MessagePack_Unpacker_finished_p(VALUE self) { UNPACKER(self, mp); @@ -240,7 +246,7 @@ static VALUE MessagePack_unpack_impl(VALUE args) size_t from = 0; char* dptr = RSTRING_PTR(data); - long dlen = RSTRING_LEN(data); + long dlen = FIX2LONG(((VALUE*)args)[2]); int ret; mp->user.origstr = data; @@ -269,7 +275,7 @@ static VALUE MessagePack_unpack_rescue(VALUE args) #endif } -static VALUE MessagePack_unpack(VALUE self, VALUE data) +static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) { CHECK_STRING_TYPE(data); msgpack_unpacker mp; @@ -278,13 +284,19 @@ static VALUE MessagePack_unpack(VALUE self, VALUE data) mp.user = ctx; rb_gc_disable(); - VALUE args[2] = {(VALUE)&mp, data}; + VALUE args[3] = {(VALUE)&mp, data, limit}; VALUE ret = rb_rescue(MessagePack_unpack_impl, (VALUE)args, MessagePack_unpack_rescue, Qnil); rb_gc_enable(); return ret; } +static VALUE MessagePack_unpack(VALUE self, VALUE data) +{ + return MessagePack_unpack_limit(self, data, + LONG2FIX(RSTRING_LEN(data))); +} + void Init_msgpack_unpack(VALUE mMessagePack) { @@ -293,10 +305,12 @@ void Init_msgpack_unpack(VALUE mMessagePack) rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); rb_define_method(cUnpacker, "initialize", MessagePack_Unpacker_initialize, 0); rb_define_method(cUnpacker, "execute", MessagePack_Unpacker_execute, 2); + rb_define_method(cUnpacker, "execute_limit", MessagePack_Unpacker_execute_limit, 3); rb_define_method(cUnpacker, "finished?", MessagePack_Unpacker_finished_p, 0); rb_define_method(cUnpacker, "data", MessagePack_Unpacker_data, 0); rb_define_method(cUnpacker, "reset", MessagePack_Unpacker_reset, 0); rb_define_module_function(mMessagePack, "unpack", MessagePack_unpack, 1); + rb_define_module_function(mMessagePack, "unpack_limit", MessagePack_unpack_limit, 2); } From 79c02cb2180809d4c2d484d534a3a2dfa014b3ba Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:01 +0000 Subject: [PATCH 0050/1172] c binding: add msgpack_object and msgpack_zone git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@97 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/object.c | 167 +++++++++++++++++++++++++++++++++++++++++++++++++++++ c/object.h | 82 ++++++++++++++++++++++++++ c/zone.c | 103 +++++++++++++++++++++++++++++++++ c/zone.h | 42 ++++++++++++++ 4 files changed, 394 insertions(+) create mode 100644 c/object.c create mode 100644 c/object.h create mode 100644 c/zone.c create mode 100644 c/zone.h diff --git a/c/object.c b/c/object.c new file mode 100644 index 0000000..7a41064 --- /dev/null +++ b/c/object.c @@ -0,0 +1,167 @@ +/* + * MessagePack for C dynamic typing routine + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/unpack.h" +#include "msgpack/unpack_define.h" +#include "msgpack/object.h" + +typedef struct { + msgpack_zone* z; + bool referenced; + bool failed; +} unpack_user; + +#define msgpack_unpack_struct(name) \ + struct msgpack_unpacker ## name + +#define msgpack_unpack_func(ret, name) \ + ret msgpack_unpacker ## name + +#define msgpack_unpack_callback(name) \ + msgpack_unpack ## name + +#define msgpack_unpack_object msgpack_object + +#define msgpack_unpack_user unpack_user + + +struct msgpack_unpacker_context; + +static void msgpack_unpacker_init(struct msgpack_unpacker_context* ctx); + +static msgpack_object msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); + +static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, + const char* data, size_t len, size_t* off); + +static inline msgpack_object msgpack_unpack_init(unpack_user* u) +{ msgpack_object o; return o; } + +static inline msgpack_object msgpack_unpack_uint8(unpack_user* u, uint8_t d) +{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + +static inline msgpack_object msgpack_unpack_uint16(unpack_user* u, uint16_t d) +{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + +static inline msgpack_object msgpack_unpack_uint32(unpack_user* u, uint32_t d) +{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + +static inline msgpack_object msgpack_unpack_uint64(unpack_user* u, uint64_t d) +{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + +static inline msgpack_object msgpack_unpack_int8(unpack_user* u, int8_t d) +{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } + +static inline msgpack_object msgpack_unpack_int16(unpack_user* u, int16_t d) +{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } + +static inline msgpack_object msgpack_unpack_int32(unpack_user* u, int32_t d) +{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } + +static inline msgpack_object msgpack_unpack_int64(unpack_user* u, int64_t d) +{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } + +static inline msgpack_object msgpack_unpack_float(unpack_user* u, float d) +{ msgpack_object o; o.type = MSGPACK_OBJECT_DOUBLE; o.via.dec = d; return o; } + +static inline msgpack_object msgpack_unpack_double(unpack_user* u, double d) +{ msgpack_object o; o.type = MSGPACK_OBJECT_DOUBLE; o.via.dec = d; return o; } + +static inline msgpack_object msgpack_unpack_nil(unpack_user* u) +{ msgpack_object o; o.type = MSGPACK_OBJECT_NIL; return o; } + +static inline msgpack_object msgpack_unpack_true(unpack_user* u) +{ msgpack_object o; o.type = MSGPACK_OBJECT_BOOLEAN; o.via.boolean = true; return o; } + +static inline msgpack_object msgpack_unpack_false(unpack_user* u) +{ msgpack_object o; o.type = MSGPACK_OBJECT_BOOLEAN; o.via.boolean = false; return o; } + +static inline msgpack_object msgpack_unpack_array(unpack_user* u, unsigned int n) +{ + msgpack_object o; + o.type = MSGPACK_OBJECT_ARRAY; + o.via.container.size = 0; + o.via.container.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); + if(o.via.container.ptr == NULL) { u->failed = true; } + return o; +} + +static inline void msgpack_unpack_array_item(unpack_user* u, msgpack_object* c, msgpack_object o) +{ + if(u->failed) { return; } + c->via.container.ptr[ c->via.container.size++ ] = o; +} + +static inline msgpack_object msgpack_unpack_map(unpack_user* u, unsigned int n) +{ + msgpack_object o; + o.type = MSGPACK_OBJECT_MAP; + o.via.container.size = 0; + o.via.container.ptr = msgpack_zone_malloc(u->z, n*2*sizeof(msgpack_object)); + if(o.via.container.ptr == NULL) { u->failed = true; } + return o; +} + +static inline void msgpack_unpack_map_item(unpack_user* u, msgpack_object* c, msgpack_object k, msgpack_object v) +{ + if(u->failed) { return; } + c->via.container.ptr[ c->via.container.size ] = k; + c->via.container.ptr[ c->via.container.size+1 ] = v; + ++c->via.container.size; +} + +static inline msgpack_object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) +{ + msgpack_object o; + o.type = MSGPACK_OBJECT_RAW; + o.via.ref.ptr = p; + o.via.ref.size = l; + u->referenced = true; + return o; +} + +#include "msgpack/unpack_template.h" + +msgpack_object_unpack_return +msgpack_object_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_object* result) +{ + struct msgpack_unpacker_context ctx; + msgpack_unpacker_init(&ctx); + unpack_user u = {z, false, false}; + ctx.user = u; + + size_t noff = (off ? *off : 0); + int ret = msgpack_unpacker_execute(&ctx, data, len, &noff); + if(ret < 0 || ctx.user.failed) { + return MSGPACK_OBJECT_PARSE_ERROR; + } else if(ret == 0) { + return MSGPACK_OBJECT_INSUFFICIENT_BYTES; + } + *result = msgpack_unpacker_data(&ctx); + if(off) { *off = noff; } + if(ret == 0) { + return MSGPACK_OBJECT_EXTRA_BYTES; + } + return MSGPACK_OBJECT_PARSE_SUCCESS; +} + + diff --git a/c/object.h b/c/object.h new file mode 100644 index 0000000..1b60c57 --- /dev/null +++ b/c/object.h @@ -0,0 +1,82 @@ +/* + * MessagePack for C dynamic typing routine + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_OBJECT_H__ +#define MSGPACK_OBJECT_H__ + +#include "msgpack/zone.h" +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef enum { + MSGPACK_OBJECT_NIL = 0x01, + MSGPACK_OBJECT_BOOLEAN = 0x02, + MSGPACK_OBJECT_POSITIVE_INTEGER = 0x03, + MSGPACK_OBJECT_NEGATIVE_INTEGER = 0x04, + MSGPACK_OBJECT_DOUBLE = 0x05, + MSGPACK_OBJECT_RAW = 0x06, + MSGPACK_OBJECT_ARRAY = 0x07, + MSGPACK_OBJECT_MAP = 0x08, +} msgpack_object_type; + +struct _msgpack_object; + +typedef union { + bool boolean; + uint64_t u64; + int64_t i64; + double dec; + struct { + struct _msgpack_object* ptr; + uint32_t size; + } container; + struct { + const char* ptr; + uint32_t size; + } ref; +} msgpack_object_union; + +typedef struct _msgpack_object { + msgpack_object_type type; + msgpack_object_union via; +} msgpack_object; + + +typedef enum { + MSGPACK_OBJECT_PARSE_SUCCESS = 0, + MSGPACK_OBJECT_EXTRA_BYTES = 1, + MSGPACK_OBJECT_INSUFFICIENT_BYTES = -1, + MSGPACK_OBJECT_PARSE_ERROR = -2, +} msgpack_object_unpack_return; + +msgpack_object_unpack_return +msgpack_object_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_object* result); + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/object.h */ + diff --git a/c/zone.c b/c/zone.c new file mode 100644 index 0000000..a41bb44 --- /dev/null +++ b/c/zone.c @@ -0,0 +1,103 @@ +/* + * MessagePack for C memory pool implementation + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/zone.h" +#include +#include + +typedef struct { + size_t free; + void* ptr; + void* alloc; +} msgpack_zone_chunk; + +struct _msgpack_zone { + msgpack_zone_chunk* array; + size_t ntail; + size_t usable; +}; + +msgpack_zone* msgpack_zone_new() +{ + return calloc(1, sizeof(msgpack_zone)); +} + +void msgpack_zone_free(msgpack_zone* z) +{ + if(z->array) { + size_t i; + for(i=0; i <= z->ntail; ++i) { + free(z->array[i].ptr); + } + } + free(z); +} + + +void* msgpack_zone_malloc(msgpack_zone* z, size_t size) +{ + if(!z->array) { + const size_t n = (sizeof(msgpack_zone_chunk) < 72/2) ? + 72 / sizeof(msgpack_zone_chunk) : 8; + msgpack_zone_chunk* array = + (msgpack_zone_chunk*)malloc(sizeof(msgpack_zone_chunk) * n); + if(!array) { return NULL; } + + size_t sz = 2048; /* FIXME chunk_size */ + while(sz < size) { sz *= 2; } + char* p = (char*)malloc(sz); + if(!p) { + free(array); + return NULL; + } + + z->array = array; + z->usable = n - 1; + array[0].free = sz - size; + array[0].ptr = p + size; + array[0].alloc = p; + return p; + } + + if(z->array[z->ntail].free > size) { + char* p = (char*)z->array[z->ntail].ptr; + z->array[z->ntail].ptr = p + size; + z->array[z->ntail].free -= size; + return p; + } + + if(z->usable <= z->ntail) { + const size_t n = (z->usable + 1) * 2; + msgpack_zone_chunk* tmp = + (msgpack_zone_chunk*)realloc(z->array, sizeof(msgpack_zone_chunk) * n); + if(!tmp) { return NULL; } + z->array = tmp; + z->usable = n - 1; + } + + size_t sz = 2048; /* FIXME chunk_size */ + while(sz < size) { sz *= 2; } + char* p = (char*)malloc(sz); + if(!p) { return NULL; } + + ++z->ntail; + z->array[z->ntail].free = sz - size; + z->array[z->ntail].ptr = p + size; + z->array[z->ntail].alloc = p; + return p; +} + diff --git a/c/zone.h b/c/zone.h new file mode 100644 index 0000000..ff6ca62 --- /dev/null +++ b/c/zone.h @@ -0,0 +1,42 @@ +/* + * MessagePack for C memory pool implementation + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_ZONE_H__ +#define MSGPACK_ZONE_H__ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +struct _msgpack_zone; +typedef struct _msgpack_zone msgpack_zone; + +msgpack_zone* msgpack_zone_new(); +void msgpack_zone_free(msgpack_zone* z); + +void* msgpack_zone_malloc(msgpack_zone* z, size_t size); + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/zone.h */ + From e3f889c9582372efec963bd7e2bd7fb618a55596 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:01 +0000 Subject: [PATCH 0051/1172] c binding: add msgpack_sbuffer git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@98 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/sbuffer.h | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 c/sbuffer.h diff --git a/c/sbuffer.h b/c/sbuffer.h new file mode 100644 index 0000000..3694dbd --- /dev/null +++ b/c/sbuffer.h @@ -0,0 +1,68 @@ +/* + * MessagePack for C simple buffer implementation + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_SBUFFER_H__ +#define MSGPACK_SBUFFER_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + char* ptr; + size_t size; + size_t capacity; +} msgpack_sbuffer; + +static inline void msgpack_sbuffer_init(msgpack_sbuffer* sbuf) +{ + memset(sbuf, 0, sizeof(msgpack_sbuffer)); +} + +static inline void msgpack_sbuffer_destroy(msgpack_sbuffer* sbuf) +{ + free(sbuf->ptr); +} + +static inline int msgpack_sbuffer_write(void* data, const char* buf, unsigned int len) +{ + msgpack_sbuffer* sbuf = (msgpack_sbuffer*)data; + if(sbuf->capacity - sbuf->size < len) { + size_t nsize = (sbuf->capacity ? sbuf->capacity*2 : 2048); + while(nsize < sbuf->size + len) { nsize *= 2; } + + void* tmp = realloc(sbuf->ptr, nsize); + if(!tmp) { return -1; } + + sbuf->ptr = (char*)tmp; + sbuf->capacity = nsize; + } + memcpy(sbuf->ptr + sbuf->size, buf, len); + sbuf->size += len; + return 0; +} + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/sbuffer.h */ + From 81a771094d2c54437ca20c3c09a5c8f05f81d0c5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:01 +0000 Subject: [PATCH 0052/1172] c binding: add msgpack::sbuffer git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@99 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/sbuffer.hpp | 86 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 cpp/sbuffer.hpp diff --git a/cpp/sbuffer.hpp b/cpp/sbuffer.hpp new file mode 100644 index 0000000..1dac6da --- /dev/null +++ b/cpp/sbuffer.hpp @@ -0,0 +1,86 @@ +// +// MessagePack for C++ simple buffer implementation +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_SBUFFER_HPP__ +#define MSGPACK_SBUFFER_HPP__ + +#include +#include +#include + +namespace msgpack { + + +class sbuffer { +public: + sbuffer() : m_capacity(0), m_size(0), m_ptr(NULL) { } + + ~sbuffer() + { + free(m_ptr); + } + +public: + void write(const char* buf, size_t len) + { + if(m_capacity - m_size < len) { + size_t nsize = (m_capacity ? m_capacity*2 : 2048); + while(nsize < m_size + len) { nsize *= 2; } + + char* tmp = (char*)realloc(m_ptr, nsize); + if(!tmp) { throw std::bad_alloc(); } + + m_ptr = tmp; + m_capacity = nsize; + } + memcpy(m_ptr + m_size, buf, len); + m_size += len; + } + + char* data() + { + return m_ptr; + } + + size_t size() const + { + return m_size; + } + + char* release() + { + char* tmp = m_ptr; + m_capacity = 0; + m_size = 0; + m_ptr = NULL; + return tmp; + } + +private: + size_t m_capacity; + size_t m_size; + char* m_ptr; + +private: + sbuffer(const sbuffer&); +}; + + +} // namespace msgpack + +#endif /* msgpack/sbuffer.hpp */ + From a114f4a5a5136a24f46bb52ffef1d1f588ba94e1 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:02 +0000 Subject: [PATCH 0053/1172] update pack/unpack routines git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@100 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/COPYING | 2 +- c/Makefile.am | 13 ++- c/README | 2 +- c/msgpack.h | 5 +- c/pack.c | 49 --------- c/pack.h | 92 +++++++++++----- c/unpack.c | 10 +- c/unpack.h | 6 +- cpp/COPYING | 2 +- cpp/Makefile.am | 5 +- cpp/README | 2 +- cpp/msgpack.hpp | 3 +- cpp/object.cpp | 2 +- cpp/object.hpp | 136 ++++++++++++++---------- cpp/pack.hpp | 216 ++++++++++++++++++++++++++++---------- cpp/type/array.hpp | 10 +- cpp/type/map.hpp | 34 +++--- cpp/type/tuple.hpp.erb | 6 +- cpp/unpack.cpp | 71 ++++++------- cpp/unpack.hpp | 2 +- cpp/zone.cpp | 2 +- cpp/zone.hpp.erb | 2 +- msgpack/pack_define.h | 2 +- msgpack/pack_template.h | 78 +++++++------- msgpack/unpack_define.h | 2 +- msgpack/unpack_template.h | 80 +++++++------- ruby/pack.c | 8 +- ruby/pack.h | 4 +- ruby/rbinit.c | 2 +- ruby/test_case.rb | 7 ++ ruby/unpack.c | 10 +- ruby/unpack.h | 4 +- 32 files changed, 503 insertions(+), 366 deletions(-) delete mode 100644 c/pack.c diff --git a/c/COPYING b/c/COPYING index 5f100cd..6f5f220 100644 --- a/c/COPYING +++ b/c/COPYING @@ -1,4 +1,4 @@ -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/c/Makefile.am b/c/Makefile.am index f1c57e9..e7cdc10 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -1,13 +1,18 @@ lib_LTLIBRARIES = libmsgpackc.la libmsgpackc_la_SOURCES = \ - pack.c \ - unpack.c + unpack.c \ + object.c \ + zone.c nobase_include_HEADERS = \ msgpack.h \ + msgpack/sbuffer.h \ msgpack/pack.h \ - msgpack/unpack.h + msgpack/unpack.h \ + msgpack/object.h \ + msgpack/zone.h -libmsgpackc_la_LDFLAGS = -version-info 0:0:0 +# -version-info CURRENT:REVISION:AGE +libmsgpackc_la_LDFLAGS = -version-info 1:0:0 diff --git a/c/README b/c/README index 609b67d..76dc221 100644 --- a/c/README +++ b/c/README @@ -4,7 +4,7 @@ MessagePack is a binary-based efficient data interchange format. -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/c/msgpack.h b/c/msgpack.h index 7a15f71..a59ef03 100644 --- a/c/msgpack.h +++ b/c/msgpack.h @@ -1,7 +1,7 @@ /* * MessagePack for C * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,5 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "msgpack/object.h" +#include "msgpack/zone.h" #include "msgpack/pack.h" #include "msgpack/unpack.h" +#include "msgpack/sbuffer.h" diff --git a/c/pack.c b/c/pack.c deleted file mode 100644 index f4787c0..0000000 --- a/c/pack.c +++ /dev/null @@ -1,49 +0,0 @@ -/* - * MessagePack packing routine for C - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "msgpack/pack.h" -#include "msgpack/pack_define.h" -#include - - -#define msgpack_pack_inline_func(name) \ - void msgpack_pack_##name - -#define msgpack_pack_inline_func_cint(name) \ - void msgpack_pack_##name - -#define msgpack_pack_user msgpack_pack_t* - -#define msgpack_pack_append_buffer(user, buf, len) \ - (*(user)->callback)((user)->data, (const char*)buf, len) - -#include "msgpack/pack_template.h" - -msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_append_buffer_t callback) -{ - msgpack_pack_t* ctx = calloc(1, sizeof(msgpack_pack_t)); - if(!ctx) { return NULL; } - ctx->data = data; - ctx->callback = callback; - return ctx; -} - -void msgpack_pack_free(msgpack_pack_t* ctx) -{ - free(ctx); -} - diff --git a/c/pack.h b/c/pack.h index ecf675c..752cbc4 100644 --- a/c/pack.h +++ b/c/pack.h @@ -1,7 +1,7 @@ /* - * MessagePack packing routine for C + * MessagePack for C packing routine * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,50 +20,90 @@ #include #include +#include #ifdef __cplusplus extern "C" { #endif -typedef void (*msgpack_pack_append_buffer_t)(void* data, const char* b, unsigned int i); +typedef int (*msgpack_pack_write_t)(void* data, const char* buf, unsigned int len); typedef struct { void* data; - msgpack_pack_append_buffer_t callback; + msgpack_pack_write_t callback; } msgpack_pack_t; -msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_append_buffer_t callback); +void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback); +msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback); void msgpack_pack_free(msgpack_pack_t* ctx); -void msgpack_pack_int(msgpack_pack_t* ctx, int d); -void msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); -void msgpack_pack_long(msgpack_pack_t* ctx, long d); -void msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); +int msgpack_pack_short(msgpack_pack_t* ctx, short d); +int msgpack_pack_int(msgpack_pack_t* ctx, int d); +int msgpack_pack_long(msgpack_pack_t* ctx, long d); +int msgpack_pack_long_long(msgpack_pack_t* ctx, long long d); +int msgpack_pack_unsigned_short(msgpack_pack_t* ctx, unsigned short d); +int msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); +int msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); +int msgpack_pack_unsigned_long_long(msgpack_pack_t* ctx, unsigned long long d); -void msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); -void msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); -void msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); -void msgpack_pack_uint64(msgpack_pack_t* ctx, uint64_t d); -void msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); -void msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); -void msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); -void msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); +int msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); +int msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); +int msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); +int msgpack_pack_uint64(msgpack_pack_t* ctx, uint64_t d); +int msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); +int msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); +int msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); +int msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); -void msgpack_pack_float(msgpack_pack_t* ctx, float d); -void msgpack_pack_double(msgpack_pack_t* ctx, double d); +int msgpack_pack_float(msgpack_pack_t* ctx, float d); +int msgpack_pack_double(msgpack_pack_t* ctx, double d); -void msgpack_pack_nil(msgpack_pack_t* ctx); -void msgpack_pack_true(msgpack_pack_t* ctx); -void msgpack_pack_false(msgpack_pack_t* ctx); +int msgpack_pack_nil(msgpack_pack_t* ctx); +int msgpack_pack_true(msgpack_pack_t* ctx); +int msgpack_pack_false(msgpack_pack_t* ctx); -void msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); +int msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); -void msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); +int msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); -void msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); -void msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); +int msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); +int msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); + + + +#define msgpack_pack_inline_func(name) \ + inline int msgpack_pack ## name + +#define msgpack_pack_inline_func_cint(name) \ + inline int msgpack_pack ## name + +#define msgpack_pack_user msgpack_pack_t* + +#define msgpack_pack_append_buffer(user, buf, len) \ + return (*(user)->callback)((user)->data, (const char*)buf, len) + +#include "msgpack/pack_template.h" + +inline void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback) +{ + ctx->data = data; + ctx->callback = callback; +} + +inline msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback) +{ + msgpack_pack_t* ctx = (msgpack_pack_t*)calloc(1, sizeof(msgpack_pack_t)); + if(!ctx) { return NULL; } + msgpack_pack_init(ctx, data, callback); + return ctx; +} + +inline void msgpack_pack_free(msgpack_pack_t* ctx) +{ + free(ctx); +} #ifdef __cplusplus diff --git a/c/unpack.c b/c/unpack.c index c7f25c4..03c67be 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -1,7 +1,7 @@ /* - * MessagePack unpacking routine for C + * MessagePack for C unpacking routine * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,13 +21,13 @@ #define msgpack_unpack_struct(name) \ - struct template_##name + struct template ## name #define msgpack_unpack_func(ret, name) \ - ret template_func_##name + ret template_func ## name #define msgpack_unpack_callback(name) \ - template_callback_##name + template_callback ## name #define msgpack_unpack_object void* diff --git a/c/unpack.h b/c/unpack.h index c1cacab..4977f51 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -1,7 +1,7 @@ /* - * MessagePack unpacking routine for C + * MessagePack for C unpacking routine * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -54,11 +54,11 @@ typedef struct { msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback); void msgpack_unpack_free(msgpack_unpack_t* ctx); -void msgpack_unpack_reset(msgpack_unpack_t* ctx); int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off); void* msgpack_unpack_data(msgpack_unpack_t* ctx); +void msgpack_unpack_reset(msgpack_unpack_t* ctx); #ifdef __cplusplus diff --git a/cpp/COPYING b/cpp/COPYING index 5f100cd..6f5f220 100644 --- a/cpp/COPYING +++ b/cpp/COPYING @@ -1,4 +1,4 @@ -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 45c75e0..c7ddf4b 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -7,6 +7,7 @@ libmsgpack_la_SOURCES = \ nobase_include_HEADERS = \ msgpack.hpp \ + msgpack/sbuffer.hpp \ msgpack/pack.hpp \ msgpack/unpack.hpp \ msgpack/object.hpp \ @@ -41,6 +42,6 @@ MOSTLYCLEANFILES = \ msgpack/type/tuple.hpp \ msgpack/zone.hpp -# FIXME -libmsgpack_la_LDFLAGS = -version-info 0:0:0 +# -version-info CURRENT:REVISION:AGE +libmsgpack_la_LDFLAGS = -version-info 1:0:0 diff --git a/cpp/README b/cpp/README index eceff1b..96f18b1 100644 --- a/cpp/README +++ b/cpp/README @@ -4,7 +4,7 @@ MessagePack is a binary-based efficient data interchange format. -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cpp/msgpack.hpp b/cpp/msgpack.hpp index 9f635d0..f00da06 100644 --- a/cpp/msgpack.hpp +++ b/cpp/msgpack.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,3 +19,4 @@ #include "msgpack/zone.hpp" #include "msgpack/pack.hpp" #include "msgpack/unpack.hpp" +#include "msgpack/sbuffer.hpp" diff --git a/cpp/object.cpp b/cpp/object.cpp index 52b6ad0..d217c06 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -1,7 +1,7 @@ // // MessagePack for C++ dynamic typed objects // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/object.hpp b/cpp/object.hpp index 1f0dcdf..c6f1e4a 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -32,14 +32,16 @@ class type_error : public std::bad_cast { }; namespace type { - static const unsigned char NIL = 0x01; - static const unsigned char BOOLEAN = 0x02; - static const unsigned char POSITIVE_INTEGER = 0x03; - static const unsigned char NEGATIVE_INTEGER = 0x04; - static const unsigned char DOUBLE = 0x05; - static const unsigned char RAW = 0x06; - static const unsigned char ARRAY = 0x07; - static const unsigned char MAP = 0x08; + enum object_type { + NIL = 0x01, + BOOLEAN = 0x02, + POSITIVE_INTEGER = 0x03, + NEGATIVE_INTEGER = 0x04, + DOUBLE = 0x05, + RAW = 0x06, + ARRAY = 0x07, + MAP = 0x08, + }; } @@ -59,7 +61,7 @@ struct object { } ref; }; - unsigned char type; + type::object_type type; union_type via; bool is_nil() { return type == type::NIL; } @@ -68,7 +70,7 @@ struct object { T as(); template - void convert(T& v); + void convert(T* v); private: struct implicit_type; @@ -77,6 +79,21 @@ public: implicit_type convert(); }; +bool operator==(const object x, const object y); +bool operator!=(const object x, const object y); + +std::ostream& operator<< (std::ostream& s, const object o); + + +template +inline void pack(Stream& s, const T& v); + +template +packer& operator<< (packer& o, const T& v); + +template +T& operator>> (object o, T& v); + struct object::implicit_type { implicit_type(object o) : obj(o) { } @@ -89,39 +106,41 @@ private: object obj; }; -std::ostream& operator<< (std::ostream& s, const object o); -bool operator==(const object x, const object y); -inline bool operator!=(const object x, const object y) { return !(x == y); } +template +class define : public Type { +public: + typedef Type msgpack_type; + typedef define define_type; + + define() {} + define(const msgpack_type& v) : msgpack_type(v) {} + + template + void msgpack_pack(Packer& o) const + { + o << static_cast(*this); + } + + void msgpack_unpack(object o) + { + o >> static_cast(*this); + } +}; -inline object& operator>> (object o, object& v) -{ - v = o; - return v; -} template -packer& operator<< (packer& o, const object& v); - - - template -inline void convert(T& v, object o) +inline packer& packer::pack(const T& v) { - o >> v; -} - -template -inline void pack(packer& o, const T& v) -{ - o << v; + *this << v; + return *this; } template inline void pack(Stream& s, const T& v) { - packer pk(s); - pack(pk, v); + packer(s).pack(v); } template @@ -130,7 +149,11 @@ inline void pack_copy(packer& o, T v) pack(o, v); } - +inline object& operator>> (object o, object& v) +{ + v = o; + return v; +} template inline T& operator>> (object o, T& v) @@ -147,27 +170,8 @@ inline packer& operator<< (packer& o, const T& v) } -template -class define : public Type { -public: - typedef Type msgpack_type; - typedef define define_type; - - define() {} - define(msgpack_type v) : msgpack_type(v) {} - - template - void msgpack_pack(Packer& o) const - { - o << static_cast(*this); - } - - void msgpack_unpack(object o) - { - o >> static_cast(*this); - } -}; - +inline bool operator!=(const object x, const object y) +{ return !(x == y); } inline object::implicit_type object::convert() @@ -179,16 +183,32 @@ template inline T object::as() { T v; - msgpack::convert(v, *this); + convert(&v); return v; } template -void object::convert(T& v) +inline void object::convert(T* v) { - msgpack::convert(v, *this); + *this >> *v; } + +// obsolete +template +inline void convert(T& v, object o) +{ + o.convert(&v); +} + +// obsolete +template +inline void pack(packer& o, const T& v) +{ + o.pack(v); +} + + template packer& operator<< (packer& o, const object& v) { diff --git a/cpp/pack.hpp b/cpp/pack.hpp index c07bcc3..52854ac 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ serializing routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -32,70 +32,73 @@ public: packer(Stream& s); public: - void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } - void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } - void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } - void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } - void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } - void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } - void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } - void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } + template + packer& pack(const T& v); - void pack_short(int d) { pack_short_impl(m_stream, d); } - void pack_int(int d) { pack_int_impl(m_stream, d); } - void pack_long(long d) { pack_long_impl(m_stream, d); } - void pack_long_long(long long d) { pack_long_long_impl(m_stream, d); } - void pack_unsigned_short(unsigned short d) { pack_unsigned_short_impl(m_stream, d); } - void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } - void pack_unsigned_long(unsigned long d) { pack_unsigned_long_impl(m_stream, d); } - void pack_unsigned_long_long(unsigned long long d) { pack_unsigned_long_long_impl(m_stream, d); } + packer& pack_uint8(uint8_t d); + packer& pack_uint16(uint16_t d); + packer& pack_uint32(uint32_t d); + packer& pack_uint64(uint64_t d); + packer& pack_int8(uint8_t d); + packer& pack_int16(uint16_t d); + packer& pack_int32(uint32_t d); + packer& pack_int64(uint64_t d); - void pack_float(float d) { pack_float_impl(m_stream, d); } - void pack_double(double d) { pack_double_impl(m_stream, d); } + packer& pack_short(int d); + packer& pack_int(int d); + packer& pack_long(long d); + packer& pack_long_long(long long d); + packer& pack_unsigned_short(unsigned short d); + packer& pack_unsigned_int(unsigned int d); + packer& pack_unsigned_long(unsigned long d); + packer& pack_unsigned_long_long(unsigned long long d); - void pack_nil() { pack_nil_impl(m_stream); } - void pack_true() { pack_true_impl(m_stream); } - void pack_false() { pack_false_impl(m_stream); } + packer& pack_float(float d); + packer& pack_double(double d); - void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } + packer& pack_nil(); + packer& pack_true(); + packer& pack_false(); - void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } + packer& pack_array(unsigned int n); - void pack_raw(size_t l) { pack_raw_impl(m_stream, l); } - void pack_raw_body(const char* b, size_t l) { pack_raw_body_impl(m_stream, b, l); } + packer& pack_map(unsigned int n); + + packer& pack_raw(size_t l); + packer& pack_raw_body(const char* b, size_t l); private: - static void pack_uint8_impl(Stream& x, uint8_t d); - static void pack_uint16_impl(Stream& x, uint16_t d); - static void pack_uint32_impl(Stream& x, uint32_t d); - static void pack_uint64_impl(Stream& x, uint64_t d); - static void pack_int8_impl(Stream& x, int8_t d); - static void pack_int16_impl(Stream& x, int16_t d); - static void pack_int32_impl(Stream& x, int32_t d); - static void pack_int64_impl(Stream& x, int64_t d); + static void _pack_uint8(Stream& x, uint8_t d); + static void _pack_uint16(Stream& x, uint16_t d); + static void _pack_uint32(Stream& x, uint32_t d); + static void _pack_uint64(Stream& x, uint64_t d); + static void _pack_int8(Stream& x, int8_t d); + static void _pack_int16(Stream& x, int16_t d); + static void _pack_int32(Stream& x, int32_t d); + static void _pack_int64(Stream& x, int64_t d); - static void pack_short_impl(Stream& x, short d); - static void pack_int_impl(Stream& x, int d); - static void pack_long_impl(Stream& x, long d); - static void pack_long_long_impl(Stream& x, long long d); - static void pack_unsigned_short_impl(Stream& x, unsigned short d); - static void pack_unsigned_int_impl(Stream& x, unsigned int d); - static void pack_unsigned_long_impl(Stream& x, unsigned long d); - static void pack_unsigned_long_long_impl(Stream& x, unsigned long long d); + static void _pack_short(Stream& x, short d); + static void _pack_int(Stream& x, int d); + static void _pack_long(Stream& x, long d); + static void _pack_long_long(Stream& x, long long d); + static void _pack_unsigned_short(Stream& x, unsigned short d); + static void _pack_unsigned_int(Stream& x, unsigned int d); + static void _pack_unsigned_long(Stream& x, unsigned long d); + static void _pack_unsigned_long_long(Stream& x, unsigned long long d); - static void pack_float_impl(Stream& x, float d); - static void pack_double_impl(Stream& x, double d); + static void _pack_float(Stream& x, float d); + static void _pack_double(Stream& x, double d); - static void pack_nil_impl(Stream& x); - static void pack_true_impl(Stream& x); - static void pack_false_impl(Stream& x); + static void _pack_nil(Stream& x); + static void _pack_true(Stream& x); + static void _pack_false(Stream& x); - static void pack_array_impl(Stream& x, unsigned int n); + static void _pack_array(Stream& x, unsigned int n); - static void pack_map_impl(Stream& x, unsigned int n); + static void _pack_map(Stream& x, unsigned int n); - static void pack_raw_impl(Stream& x, size_t l); - static void pack_raw_body_impl(Stream& x, const void* b, size_t l); + static void _pack_raw(Stream& x, size_t l); + static void _pack_raw_body(Stream& x, const void* b, size_t l); static void append_buffer(Stream& x, const unsigned char* buf, unsigned int len) { x.write((const char*)buf, len); } @@ -107,13 +110,14 @@ private: packer(); }; + #define msgpack_pack_inline_func(name) \ template \ - inline void packer::pack_ ## name ## _impl + inline void packer::_pack ## name #define msgpack_pack_inline_func_cint(name) \ template \ - inline void packer::pack_ ## name ## _impl + inline void packer::_pack ## name #define msgpack_pack_user Stream& @@ -125,6 +129,112 @@ private: template packer::packer(Stream& s) : m_stream(s) { } +template +inline packer& packer::pack_uint8(uint8_t d) +{ _pack_uint8(m_stream, d); return *this; } + +template +inline packer& packer::pack_uint16(uint16_t d) +{ _pack_uint16(m_stream, d); return *this; } + +template +inline packer& packer::pack_uint32(uint32_t d) +{ _pack_uint32(m_stream, d); return *this; } + +template +inline packer& packer::pack_uint64(uint64_t d) +{ _pack_uint64(m_stream, d); return *this; } + +template +inline packer& packer::pack_int8(uint8_t d) +{ _pack_int8(m_stream, d); return *this; } + +template +inline packer& packer::pack_int16(uint16_t d) +{ _pack_int16(m_stream, d); return *this; } + +template +inline packer& packer::pack_int32(uint32_t d) +{ _pack_int32(m_stream, d); return *this; } + +template +inline packer& packer::pack_int64(uint64_t d) +{ _pack_int64(m_stream, d); return *this;} + + +template +inline packer& packer::pack_short(int d) +{ _pack_short(m_stream, d); return *this; } + +template +inline packer& packer::pack_int(int d) +{ _pack_int(m_stream, d); return *this; } + +template +inline packer& packer::pack_long(long d) +{ _pack_long(m_stream, d); return *this; } + +template +inline packer& packer::pack_long_long(long long d) +{ _pack_long_long(m_stream, d); return *this; } + +template +inline packer& packer::pack_unsigned_short(unsigned short d) +{ _pack_unsigned_short(m_stream, d); return *this; } + +template +inline packer& packer::pack_unsigned_int(unsigned int d) +{ _pack_unsigned_int(m_stream, d); return *this; } + +template +inline packer& packer::pack_unsigned_long(unsigned long d) +{ _pack_unsigned_long(m_stream, d); return *this; } + +template +inline packer& packer::pack_unsigned_long_long(unsigned long long d) +{ _pack_unsigned_long_long(m_stream, d); return *this; } + + +template +inline packer& packer::pack_float(float d) +{ _pack_float(m_stream, d); return *this; } + +template +inline packer& packer::pack_double(double d) +{ _pack_double(m_stream, d); return *this; } + + +template +inline packer& packer::pack_nil() +{ _pack_nil(m_stream); return *this; } + +template +inline packer& packer::pack_true() +{ _pack_true(m_stream); return *this; } + +template +inline packer& packer::pack_false() +{ _pack_false(m_stream); return *this; } + + +template +inline packer& packer::pack_array(unsigned int n) +{ _pack_array(m_stream, n); return *this; } + + +template +inline packer& packer::pack_map(unsigned int n) +{ _pack_map(m_stream, n); return *this; } + + +template +inline packer& packer::pack_raw(size_t l) +{ _pack_raw(m_stream, l); return *this; } + +template +inline packer& packer::pack_raw_body(const char* b, size_t l) +{ _pack_raw_body(m_stream, b, l); return *this; } + } // namespace msgpack diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp index 0522d4c..f9f8038 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/array.hpp @@ -29,11 +29,11 @@ inline std::vector operator>> (object o, std::vector& v) { if(o.type != type::ARRAY) { throw type_error(); } v.resize(o.via.container.size); - object* p(o.via.container.ptr); - object* const pend(o.via.container.ptr + o.via.container.size); - T* it(&v.front()); + object* p = o.via.container.ptr; + object* const pend = o.via.container.ptr + o.via.container.size; + T* it = &v.front(); for(; p < pend; ++p, ++it) { - convert(*it, *p); + p->convert(it); } return v; } @@ -45,7 +45,7 @@ inline packer& operator<< (packer& o, const std::vector& v) o.pack_array(v.size()); for(typename std::vector::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(o, *it); + o.pack(*it); } return o; } diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index c79f31c..619c36e 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -47,12 +47,12 @@ inline type::assoc_vector& operator>> (object o, type::assoc_vector& v { if(o.type != type::MAP) { throw type_error(); } v.resize(o.via.container.size); - object* p(o.via.container.ptr); - object* const pend(o.via.container.ptr + o.via.container.size); + object* p = o.via.container.ptr; + object* const pend = o.via.container.ptr + o.via.container.size; std::pair* it(&v.front()); for(; p < pend; ++it) { - convert(it->first, *p); ++p; - convert(it->second, *p); ++p; + p->convert(&it->first); ++p; + p->convert(&it->second); ++p; } std::sort(v.begin(), v.end(), type::detail::pair_first_less()); return v; @@ -64,8 +64,8 @@ inline packer& operator<< (packer& o, const type::assoc_vector::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(o, it->first); - pack(o, it->second); + o.pack(it->first); + o.pack(it->second); } return o; } @@ -79,14 +79,14 @@ inline std::map operator>> (object o, std::map& v) object* const pend(o.via.container.ptr + o.via.container.size*2); while(p < pend) { K key; - convert(key, *p); ++p; + p->convert(&key); ++p; typename std::map::iterator it(v.find(key)); if(it != v.end()) { V val; - convert(val, *p); ++p; + p->convert(&val); ++p; it->insert( std::pair(key, val) ); } else { - convert(it->second, *p); ++p; + p->convert(&it->second); ++p; } } return v; @@ -98,8 +98,8 @@ inline packer& operator<< (packer& o, const std::map& v) o.pack_map(v.size()); for(typename std::map::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(o, it->first); - pack(o, it->second); + o.pack(it->first); + o.pack(it->second); } return o; } @@ -109,12 +109,12 @@ template inline std::multimap operator>> (object o, std::multimap& v) { if(o.type != type::MAP) { throw type_error(); } - object* p(o.via.container.ptr); - object* const pend(o.via.container.ptr + o.via.container.size*2); + object* p = o.via.container.ptr; + object* const pend = o.via.container.ptr + o.via.container.size*2; while(p < pend) { std::pair value; - convert(value.first, *p); ++p; - convert(value.second, *p); ++p; + p->convert(&value.first); ++p; + p->convert(&value.second); ++p; v.insert(value); } return v; @@ -126,8 +126,8 @@ inline packer& operator<< (packer& o, const std::multimap& o.pack_multimap(v.size()); for(typename std::multimap::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(o, it->first); - pack(o, it->second); + o.pack(it->first); + o.pack(it->second); } return o; } diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index 13d4bd7..a66c57e 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -111,7 +111,7 @@ struct tuple, A<%=j%><%}%>> { tuple() {} tuple(typename tuple_type::transparent_reference _a0<%1.upto(i) {|j|%>, typename tuple_type>::transparent_reference _a<%=j%><%}%>) : a0(_a0)<%1.upto(i) {|j|%>, a<%=j%>(_a<%=j%>)<%}%> {} - tuple(object o) { convert(*this, o); } + tuple(object o) { o.convert(this); } template typename tuple_element::reference get() { return tuple_element(*this).get(); } template typename const_tuple_element::const_reference get() const @@ -138,7 +138,7 @@ type::tuple, A<%=j%><%}%>>& operator>> ( if(o.type != type::ARRAY) { throw type_error(); } if(o.via.container.size < <%=i+1%>) { throw type_error(); } <%0.upto(i) {|j|%> - convert>(v.template get<<%=j%>>(), o.via.container.ptr[<%=j%>]);<%}%> + o.via.container.ptr[<%=j%>].convert>(&v.template get<<%=j%>>());<%}%> return v; } <%}%> @@ -158,7 +158,7 @@ const packer& operator<< ( const type::tuple, A<%=j%><%}%>>& v) { o.pack_array(<%=i+1%>); <%0.upto(i) {|j|%> - pack(o, v.template get<<%=j%>>());<%}%> + o.pack(v.template get<<%=j%>>());<%}%> return o; } <%}%> diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index f2ef330..62088d6 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -1,7 +1,7 @@ // // MessagePack for C++ deserializing routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,30 +23,25 @@ namespace msgpack { //namespace { -struct allocator { +struct unpack_user { zone* z; bool referenced; - - inline object* malloc_object(size_t n) - { - return (object*)z->malloc(sizeof(object)*n); - } }; //} // noname namespace #define msgpack_unpack_struct(name) \ - struct msgpack_unpacker_##name + struct msgpack_unpacker ## name #define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker_##name + ret msgpack_unpacker ## name #define msgpack_unpack_callback(name) \ - msgpack_unpack_##name + msgpack_unpack ## name #define msgpack_unpack_object object -#define msgpack_unpack_user allocator +#define msgpack_unpack_user unpack_user struct msgpack_unpacker_context; @@ -59,87 +54,87 @@ static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, const char* data, size_t len, size_t* off); -static inline object msgpack_unpack_init(allocator* a) +static inline object msgpack_unpack_init(unpack_user* u) { return object(); } -static inline object msgpack_unpack_uint8(allocator* a, uint8_t d) +static inline object msgpack_unpack_uint8(unpack_user* u, uint8_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_uint16(allocator* a, uint16_t d) +static inline object msgpack_unpack_uint16(unpack_user* u, uint16_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_uint32(allocator* a, uint32_t d) +static inline object msgpack_unpack_uint32(unpack_user* u, uint32_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_uint64(allocator* a, uint64_t d) +static inline object msgpack_unpack_uint64(unpack_user* u, uint64_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_int8(allocator* a, int8_t d) +static inline object msgpack_unpack_int8(unpack_user* u, int8_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_int16(allocator* a, int16_t d) +static inline object msgpack_unpack_int16(unpack_user* u, int16_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_int32(allocator* a, int32_t d) +static inline object msgpack_unpack_int32(unpack_user* u, int32_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_int64(allocator* a, int64_t d) +static inline object msgpack_unpack_int64(unpack_user* u, int64_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_float(allocator* a, float d) +static inline object msgpack_unpack_float(unpack_user* u, float d) { object o; o.type = type::DOUBLE; o.via.dec = d; return o; } -static inline object msgpack_unpack_double(allocator* a, double d) +static inline object msgpack_unpack_double(unpack_user* u, double d) { object o; o.type = type::DOUBLE; o.via.dec = d; return o; } -static inline object msgpack_unpack_nil(allocator* a) +static inline object msgpack_unpack_nil(unpack_user* u) { object o; o.type = type::NIL; return o; } -static inline object msgpack_unpack_true(allocator* a) +static inline object msgpack_unpack_true(unpack_user* u) { object o; o.type = type::BOOLEAN; o.via.boolean = true; return o; } -static inline object msgpack_unpack_false(allocator* a) +static inline object msgpack_unpack_false(unpack_user* u) { object o; o.type = type::BOOLEAN; o.via.boolean = false; return o; } -static inline object msgpack_unpack_array(allocator* a, unsigned int n) +static inline object msgpack_unpack_array(unpack_user* u, unsigned int n) { object o; o.type = type::ARRAY; o.via.container.size = 0; - o.via.container.ptr = a->malloc_object(n); + o.via.container.ptr = (object*)u->z->malloc(n*sizeof(object)); return o; } -static inline void msgpack_unpack_array_item(allocator* a, object* c, object o) +static inline void msgpack_unpack_array_item(unpack_user* u, object* c, object o) { c->via.container.ptr[ c->via.container.size++ ] = o; } -static inline object msgpack_unpack_map(allocator* a, unsigned int n) +static inline object msgpack_unpack_map(unpack_user* u, unsigned int n) { object o; o.type = type::MAP; o.via.container.size = 0; - o.via.container.ptr = a->malloc_object(n*2); + o.via.container.ptr = (object*)u->z->malloc(n*2*sizeof(object)); return o; } -static inline void msgpack_unpack_map_item(allocator* a, object* c, object k, object v) +static inline void msgpack_unpack_map_item(unpack_user* u, object* c, object k, object v) { c->via.container.ptr[ c->via.container.size ] = k; c->via.container.ptr[ c->via.container.size+1 ] = v; ++c->via.container.size; } -static inline object msgpack_unpack_raw(allocator* a, const char* b, const char* p, unsigned int l) +static inline object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) { object o; o.type = type::RAW; o.via.ref.ptr = p; o.via.ref.size = l; - a->referenced = true; + u->referenced = true; return o; } @@ -151,8 +146,8 @@ struct context { context() { msgpack_unpacker_init(&m_ctx); - allocator a = {NULL, false}; - m_ctx.user = a; + unpack_user u = {NULL, false}; + m_ctx.user = u; } ~context() { } @@ -171,8 +166,8 @@ struct context { { zone* z = m_ctx.user.z; msgpack_unpacker_init(&m_ctx); - allocator a = {z, false}; - m_ctx.user = a; + unpack_user u = {z, false}; + m_ctx.user = u; } void set_zone(zone* z) @@ -193,7 +188,7 @@ private: context(const context&); }; -context* as_ctx(void* m) +static inline context* as_ctx(void* m) { return reinterpret_cast(m); } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 293a5d3..cde45e7 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ deserializing routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/zone.cpp b/cpp/zone.cpp index 7c5a1c9..f765266 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -1,7 +1,7 @@ // // MessagePack for C++ memory pool // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 9a8c81a..c0eb2e3 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -1,7 +1,7 @@ // // MessagePack for C++ memory pool // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h index 6512d77..33408e5 100644 --- a/msgpack/pack_define.h +++ b/msgpack/pack_define.h @@ -1,7 +1,7 @@ /* * MessagePack unpacking routine template * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index dafb0b1..212c0e6 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -1,7 +1,7 @@ /* * MessagePack packing routine template * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -278,49 +278,49 @@ do { \ #ifdef msgpack_pack_inline_func_fastint -msgpack_pack_inline_func_fastint(uint8)(msgpack_pack_user x, uint8_t d) +msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) { const unsigned char buf[2] = {0xcc, d}; msgpack_pack_append_buffer(x, buf, 2); } -msgpack_pack_inline_func_fastint(uint16)(msgpack_pack_user x, uint16_t d) +msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func_fastint(uint32)(msgpack_pack_user x, uint32_t d) +msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { const unsigned char buf[5] = {0xce, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func_fastint(uint64)(msgpack_pack_user x, uint64_t d) +msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } -msgpack_pack_inline_func_fastint(int8)(msgpack_pack_user x, int8_t d) +msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) { const unsigned char buf[2] = {0xd0, d}; msgpack_pack_append_buffer(x, buf, 2); } -msgpack_pack_inline_func_fastint(int16)(msgpack_pack_user x, int16_t d) +msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func_fastint(int32)(msgpack_pack_user x, int32_t d) +msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func_fastint(int64)(msgpack_pack_user x, int64_t d) +msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); @@ -330,42 +330,42 @@ msgpack_pack_inline_func_fastint(int64)(msgpack_pack_user x, int64_t d) #endif -msgpack_pack_inline_func(uint8)(msgpack_pack_user x, uint8_t d) +msgpack_pack_inline_func(_uint8)(msgpack_pack_user x, uint8_t d) { msgpack_pack_real_uint8(x, d); } -msgpack_pack_inline_func(uint16)(msgpack_pack_user x, uint16_t d) +msgpack_pack_inline_func(_uint16)(msgpack_pack_user x, uint16_t d) { msgpack_pack_real_uint16(x, d); } -msgpack_pack_inline_func(uint32)(msgpack_pack_user x, uint32_t d) +msgpack_pack_inline_func(_uint32)(msgpack_pack_user x, uint32_t d) { msgpack_pack_real_uint32(x, d); } -msgpack_pack_inline_func(uint64)(msgpack_pack_user x, uint64_t d) +msgpack_pack_inline_func(_uint64)(msgpack_pack_user x, uint64_t d) { msgpack_pack_real_uint64(x, d); } -msgpack_pack_inline_func(int8)(msgpack_pack_user x, int8_t d) +msgpack_pack_inline_func(_int8)(msgpack_pack_user x, int8_t d) { msgpack_pack_real_int8(x, d); } -msgpack_pack_inline_func(int16)(msgpack_pack_user x, int16_t d) +msgpack_pack_inline_func(_int16)(msgpack_pack_user x, int16_t d) { msgpack_pack_real_int16(x, d); } -msgpack_pack_inline_func(int32)(msgpack_pack_user x, int32_t d) +msgpack_pack_inline_func(_int32)(msgpack_pack_user x, int32_t d) { msgpack_pack_real_int32(x, d); } -msgpack_pack_inline_func(int64)(msgpack_pack_user x, int64_t d) +msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d) { msgpack_pack_real_int64(x, d); } @@ -373,7 +373,7 @@ msgpack_pack_inline_func(int64)(msgpack_pack_user x, int64_t d) #ifdef msgpack_pack_inline_func_cint -msgpack_pack_inline_func_cint(short)(msgpack_pack_user x, short d) +msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d) { #if defined(SIZEOF_SHORT) || defined(SHRT_MAX) #if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff @@ -394,7 +394,7 @@ if(sizeof(short) == 2) { #endif } -msgpack_pack_inline_func_cint(int)(msgpack_pack_user x, int d) +msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d) { #if defined(SIZEOF_INT) || defined(INT_MAX) #if SIZEOF_INT == 2 || INT_MAX == 0x7fff @@ -415,7 +415,7 @@ if(sizeof(int) == 2) { #endif } -msgpack_pack_inline_func_cint(long)(msgpack_pack_user x, long d) +msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d) { #if defined(SIZEOF_LONG) || defined(LONG_MAX) #if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL @@ -436,7 +436,7 @@ if(sizeof(long) == 2) { #endif } -msgpack_pack_inline_func_cint(long_long)(msgpack_pack_user x, long long d) +msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d) { #if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX) #if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL @@ -457,7 +457,7 @@ if(sizeof(long long) == 2) { #endif } -msgpack_pack_inline_func_cint(unsigned_short)(msgpack_pack_user x, unsigned short d) +msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d) { #if defined(SIZEOF_SHORT) || defined(USHRT_MAX) #if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU @@ -478,7 +478,7 @@ if(sizeof(unsigned short) == 2) { #endif } -msgpack_pack_inline_func_cint(unsigned_int)(msgpack_pack_user x, unsigned int d) +msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d) { #if defined(SIZEOF_INT) || defined(UINT_MAX) #if SIZEOF_INT == 2 || UINT_MAX == 0xffffU @@ -499,7 +499,7 @@ if(sizeof(unsigned int) == 2) { #endif } -msgpack_pack_inline_func_cint(unsigned_long)(msgpack_pack_user x, unsigned long d) +msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d) { #if defined(SIZEOF_LONG) || defined(ULONG_MAX) #if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL @@ -520,7 +520,7 @@ if(sizeof(unsigned int) == 2) { #endif } -msgpack_pack_inline_func_cint(unsigned_long_long)(msgpack_pack_user x, unsigned long long d) +msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d) { #if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX) #if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL @@ -550,17 +550,19 @@ if(sizeof(unsigned long long) == 2) { * Float */ -msgpack_pack_inline_func(float)(msgpack_pack_user x, float d) +msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { - uint32_t n = *((uint32_t*)&d); // FIXME - const unsigned char buf[5] = {0xca, STORE_BE32(n)}; + union { char buf[4]; uint32_t num; } f; + *((float*)&f.buf) = d; // FIXME + const unsigned char buf[5] = {0xca, STORE_BE32(f.num)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func(double)(msgpack_pack_user x, double d) +msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { - uint64_t n = *((uint64_t*)&d); // FIXME - const unsigned char buf[9] = {0xcb, STORE_BE64(n)}; + union { char buf[8]; uint64_t num; } f; + *((double*)&f.buf) = d; // FIXME + const unsigned char buf[9] = {0xcb, STORE_BE64(f.num)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -569,7 +571,7 @@ msgpack_pack_inline_func(double)(msgpack_pack_user x, double d) * Nil */ -msgpack_pack_inline_func(nil)(msgpack_pack_user x) +msgpack_pack_inline_func(_nil)(msgpack_pack_user x) { static const unsigned char d = 0xc0; msgpack_pack_append_buffer(x, &d, 1); @@ -580,13 +582,13 @@ msgpack_pack_inline_func(nil)(msgpack_pack_user x) * Boolean */ -msgpack_pack_inline_func(true)(msgpack_pack_user x) +msgpack_pack_inline_func(_true)(msgpack_pack_user x) { static const unsigned char d = 0xc3; msgpack_pack_append_buffer(x, &d, 1); } -msgpack_pack_inline_func(false)(msgpack_pack_user x) +msgpack_pack_inline_func(_false)(msgpack_pack_user x) { static const unsigned char d = 0xc2; msgpack_pack_append_buffer(x, &d, 1); @@ -597,7 +599,7 @@ msgpack_pack_inline_func(false)(msgpack_pack_user x) * Array */ -msgpack_pack_inline_func(array)(msgpack_pack_user x, unsigned int n) +msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x90 | n; @@ -618,7 +620,7 @@ msgpack_pack_inline_func(array)(msgpack_pack_user x, unsigned int n) * Map */ -msgpack_pack_inline_func(map)(msgpack_pack_user x, unsigned int n) +msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; @@ -639,7 +641,7 @@ msgpack_pack_inline_func(map)(msgpack_pack_user x, unsigned int n) * Raw */ -msgpack_pack_inline_func(raw)(msgpack_pack_user x, size_t l) +msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; @@ -655,7 +657,7 @@ msgpack_pack_inline_func(raw)(msgpack_pack_user x, size_t l) } } -msgpack_pack_inline_func(raw_body)(msgpack_pack_user x, const void* b, size_t l) +msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l) { msgpack_pack_append_buffer(x, (const unsigned char*)b, l); } diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index f639841..63668c2 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -1,7 +1,7 @@ /* * MessagePack unpacking routine template * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index a6d7dcb..a45369b 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -1,7 +1,7 @@ /* * MessagePack unpacking routine template * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,38 +41,38 @@ #endif -msgpack_unpack_struct_decl(stack) { +msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; size_t count; unsigned int ct; msgpack_unpack_object map_key; }; -msgpack_unpack_struct_decl(context) { +msgpack_unpack_struct_decl(_context) { msgpack_unpack_user user; // must be first unsigned int cs; unsigned int trail; unsigned int top; - msgpack_unpack_struct(stack) stack[MSGPACK_MAX_STACK_SIZE]; + msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE]; }; -msgpack_unpack_func(void, init)(msgpack_unpack_struct(context)* ctx) +msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) { - /*memset(ctx, 0, sizeof( msgpack_unpack_struct(context) )); FIXME needed? */ + /*memset(ctx, 0, sizeof( msgpack_unpack_struct(_context) )); FIXME needed? */ ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; - ctx->stack[0].obj = msgpack_unpack_callback(init)(&ctx->user); + ctx->stack[0].obj = msgpack_unpack_callback(_init)(&ctx->user); } -msgpack_unpack_func(msgpack_unpack_object, data)(msgpack_unpack_struct(context)* unpacker) +msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* unpacker) { return (unpacker)->stack[0].obj; } -msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const char* data, size_t len, size_t* off) +msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) { assert(len >= *off); @@ -83,11 +83,11 @@ msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const cha unsigned int trail = ctx->trail; unsigned int cs = ctx->cs; unsigned int top = ctx->top; - msgpack_unpack_struct(stack)* stack = ctx->stack; + msgpack_unpack_struct(_stack)* stack = ctx->stack; msgpack_unpack_user* user = &ctx->user; msgpack_unpack_object obj; - msgpack_unpack_struct(stack)* c = NULL; + msgpack_unpack_struct(_stack)* c = NULL; int ret; @@ -139,19 +139,19 @@ msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const cha case CS_HEADER: switch(*p) { case 0x00 ... 0x7f: // Positive Fixnum - push_fixed_value(uint8, *(uint8_t*)p); + push_fixed_value(_uint8, *(uint8_t*)p); case 0xe0 ... 0xff: // Negative Fixnum - push_fixed_value(int8, *(int8_t*)p); + push_fixed_value(_int8, *(int8_t*)p); case 0xc0 ... 0xdf: // Variable switch(*p) { case 0xc0: // nil - push_simple_value(nil); + push_simple_value(_nil); //case 0xc1: // string // again_terminal_trail(NEXT_CS(p), p+1); case 0xc2: // false - push_simple_value(false); + push_simple_value(_false); case 0xc3: // true - push_simple_value(true); + push_simple_value(_true); //case 0xc4: //case 0xc5: //case 0xc6: @@ -188,9 +188,9 @@ msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const cha case 0xa0 ... 0xbf: // FixRaw again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); case 0x90 ... 0x9f: // FixArray - start_container(array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); + start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); case 0x80 ... 0x8f: // FixMap - start_container(map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); + start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); default: goto _failed; @@ -208,28 +208,30 @@ msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const cha //case CS_ //case CS_ case CS_FLOAT: { - uint32_t x = PTR_CAST_32(n); // FIXME - push_fixed_value(float, *((float*)&x)); } + union { uint32_t num; char buf[4]; } f; + f.num = PTR_CAST_32(n); // FIXME + push_fixed_value(_float, *((float*)f.buf)); } case CS_DOUBLE: { - uint64_t x = PTR_CAST_64(n); // FIXME - push_fixed_value(double, *((double*)&x)); } + union { uint64_t num; char buf[8]; } f; + f.num = PTR_CAST_64(n); // FIXME + push_fixed_value(_double, *((double*)f.buf)); } case CS_UINT_8: - push_fixed_value(uint8, (uint8_t)PTR_CAST_8(n)); + push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n)); case CS_UINT_16: - push_fixed_value(uint16, (uint16_t)PTR_CAST_16(n)); + push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n)); case CS_UINT_32: - push_fixed_value(uint32, (uint32_t)PTR_CAST_32(n)); + push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n)); case CS_UINT_64: - push_fixed_value(uint64, (uint64_t)PTR_CAST_64(n)); + push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n)); case CS_INT_8: - push_fixed_value(int8, (int8_t)PTR_CAST_8(n)); + push_fixed_value(_int8, (int8_t)PTR_CAST_8(n)); case CS_INT_16: - push_fixed_value(int16, (int16_t)PTR_CAST_16(n)); + push_fixed_value(_int16, (int16_t)PTR_CAST_16(n)); case CS_INT_32: - push_fixed_value(int32, (int32_t)PTR_CAST_32(n)); + push_fixed_value(_int32, (int32_t)PTR_CAST_32(n)); case CS_INT_64: - push_fixed_value(int64, (int64_t)PTR_CAST_64(n)); + push_fixed_value(_int64, (int64_t)PTR_CAST_64(n)); //case CS_ //case CS_ @@ -240,7 +242,7 @@ msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const cha //case ACS_BIG_INT_VALUE: //_big_int_zero: // // FIXME - // push_variable_value(big_int, data, n, trail); + // push_variable_value(_big_int, data, n, trail); //case CS_BIG_FLOAT_16: // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); @@ -249,7 +251,7 @@ msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const cha //case ACS_BIG_FLOAT_VALUE: //_big_float_zero: // // FIXME - // push_variable_value(big_float, data, n, trail); + // push_variable_value(_big_float, data, n, trail); case CS_RAW_16: again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); @@ -257,17 +259,17 @@ msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const cha again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); case ACS_RAW_VALUE: _raw_zero: - push_variable_value(raw, data, n, trail); + push_variable_value(_raw, data, n, trail); case CS_ARRAY_16: - start_container(array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); case CS_ARRAY_32: - start_container(array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); case CS_MAP_16: - start_container(map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); case CS_MAP_32: - start_container(map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); default: goto _failed; @@ -279,7 +281,7 @@ _push: c = &stack[top-1]; switch(c->ct) { case CT_ARRAY_ITEM: - msgpack_unpack_callback(array_item)(user, &c->obj, obj); + msgpack_unpack_callback(_array_item)(user, &c->obj, obj); if(--c->count == 0) { obj = c->obj; --top; @@ -292,7 +294,7 @@ _push: c->ct = CT_MAP_VALUE; goto _header_again; case CT_MAP_VALUE: - msgpack_unpack_callback(map_item)(user, &c->obj, c->map_key, obj); + msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj); if(--c->count == 0) { obj = c->obj; --top; diff --git a/ruby/pack.c b/ruby/pack.c index c399c25..0a3711f 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -1,7 +1,7 @@ /* - * MessagePack packing routine for Ruby + * MessagePack for Ruby packing routine * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,10 +19,10 @@ #include "msgpack/pack_define.h" #define msgpack_pack_inline_func(name) \ - static inline void msgpack_pack_##name + static inline void msgpack_pack ## name #define msgpack_pack_inline_func_cint(name) \ - static inline void msgpack_pack_##name + static inline void msgpack_pack ## name #define msgpack_pack_user VALUE diff --git a/ruby/pack.h b/ruby/pack.h index c38ac48..c9b08a4 100644 --- a/ruby/pack.h +++ b/ruby/pack.h @@ -1,7 +1,7 @@ /* - * MessagePack packing routine for Ruby + * MessagePack for Ruby packing routine * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ruby/rbinit.c b/ruby/rbinit.c index 7ef92fb..80d1d8c 100644 --- a/ruby/rbinit.c +++ b/ruby/rbinit.c @@ -1,7 +1,7 @@ /* * MessagePack for Ruby * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ruby/test_case.rb b/ruby/test_case.rb index af6d160..1e2d474 100644 --- a/ruby/test_case.rb +++ b/ruby/test_case.rb @@ -75,6 +75,13 @@ class MessagePackTestFormat < Test::Unit::TestCase check 9, -(1<<63) end + it "double" do + check 9, 1.0 + check 9, 0.1 + check 9, -0.1 + check 9, -1.0 + end + it "fixraw" do check_raw 1, 0 check_raw 1, (1<<5)-1 diff --git a/ruby/unpack.c b/ruby/unpack.c index 2920240..4650ed1 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -1,7 +1,7 @@ /* - * MessagePack unpacking routine for Ruby + * MessagePack for Ruby unpacking routine * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,13 +26,13 @@ typedef struct { #define msgpack_unpack_struct(name) \ - struct msgpack_unpacker_##name + struct msgpack_unpacker ## name #define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker_##name + ret msgpack_unpacker ## name #define msgpack_unpack_callback(name) \ - template_callback_##name + template_callback ## name #define msgpack_unpack_object VALUE diff --git a/ruby/unpack.h b/ruby/unpack.h index 0fe01ec..ce2a8de 100644 --- a/ruby/unpack.h +++ b/ruby/unpack.h @@ -1,7 +1,7 @@ /* - * MessagePack unpacking routine for Ruby + * MessagePack for Ruby unpacking routine * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 01d40bbb765a30d32c24298a2b666152de8a283f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:02 +0000 Subject: [PATCH 0054/1172] fix c/configure.in git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@101 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c/configure.in b/c/configure.in index 0017917..47eb8aa 100644 --- a/c/configure.in +++ b/c/configure.in @@ -1,4 +1,4 @@ -AC_INIT(pack.c) +AC_INIT(unpack.c) AM_INIT_AUTOMAKE(msgpackc, 0.1.0) AC_CONFIG_HEADER(config.h) From 6083300ea949855e1b2cd3cc4e92bf340c4e90e7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:02 +0000 Subject: [PATCH 0055/1172] ruby binding: simplify gem package skeleton git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@102 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- ruby/gem/{History.txt => ChangeLog} | 0 ruby/gem/License.txt | 13 - ruby/gem/Manifest.txt | 23 - ruby/gem/PostInstall.txt | 1 - ruby/gem/README | 29 + ruby/gem/README.txt | 23 - ruby/gem/Rakefile | 137 ++- ruby/gem/config/hoe.rb | 75 -- ruby/gem/config/requirements.rb | 15 - ruby/gem/lib/msgpack/version.rb | 9 - ruby/gem/script/console | 10 - ruby/gem/script/destroy | 14 - ruby/gem/script/generate | 14 - ruby/gem/script/txt2html | 82 -- ruby/gem/setup.rb | 1585 --------------------------- ruby/gem/tasks/deployment.rake | 34 - ruby/gem/tasks/environment.rake | 7 - ruby/gem/test/test_helper.rb | 3 + ruby/gengem.sh | 1 + ruby/test_case.rb | 3 +- 20 files changed, 168 insertions(+), 1910 deletions(-) rename ruby/gem/{History.txt => ChangeLog} (100%) delete mode 100644 ruby/gem/License.txt delete mode 100644 ruby/gem/Manifest.txt delete mode 100644 ruby/gem/PostInstall.txt create mode 100644 ruby/gem/README delete mode 100644 ruby/gem/README.txt delete mode 100644 ruby/gem/config/hoe.rb delete mode 100644 ruby/gem/config/requirements.rb delete mode 100644 ruby/gem/lib/msgpack/version.rb delete mode 100755 ruby/gem/script/console delete mode 100755 ruby/gem/script/destroy delete mode 100755 ruby/gem/script/generate delete mode 100755 ruby/gem/script/txt2html delete mode 100644 ruby/gem/setup.rb delete mode 100644 ruby/gem/tasks/deployment.rake delete mode 100644 ruby/gem/tasks/environment.rake create mode 100644 ruby/gem/test/test_helper.rb diff --git a/ruby/gem/History.txt b/ruby/gem/ChangeLog similarity index 100% rename from ruby/gem/History.txt rename to ruby/gem/ChangeLog diff --git a/ruby/gem/License.txt b/ruby/gem/License.txt deleted file mode 100644 index f4000b7..0000000 --- a/ruby/gem/License.txt +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2008 Furuhashi Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/ruby/gem/Manifest.txt b/ruby/gem/Manifest.txt deleted file mode 100644 index 17e2b78..0000000 --- a/ruby/gem/Manifest.txt +++ /dev/null @@ -1,23 +0,0 @@ -License.txt -Manifest.txt -README.txt -Rakefile -config/hoe.rb -config/requirements.rb -ext/extconf.rb -ext/pack.c -ext/pack.h -ext/rbinit.c -ext/unpack.c -ext/unpack.h -msgpack/pack_define.h -msgpack/pack_template.h -msgpack/unpack_define.h -msgpack/unpack_template.h -lib/msgpack/version.rb -script/console -script/destroy -script/generate -setup.rb -tasks/deployment.rake -tasks/environment.rake diff --git a/ruby/gem/PostInstall.txt b/ruby/gem/PostInstall.txt deleted file mode 100644 index 8b13789..0000000 --- a/ruby/gem/PostInstall.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ruby/gem/README b/ruby/gem/README new file mode 100644 index 0000000..78fe6cb --- /dev/null +++ b/ruby/gem/README @@ -0,0 +1,29 @@ + += MessagePack + + +== Description + + +== Installation + +=== Archive Installation + + rake install + +=== Gem Installation + + gem install msgpack + + +== Features/Problems + + +== Synopsis + + +== Copyright + +Author:: frsyuki +Copyright:: Copyright (c) 2009 frsyuki +License:: Apache License, Version 2.0 diff --git a/ruby/gem/README.txt b/ruby/gem/README.txt deleted file mode 100644 index 7c24b8f..0000000 --- a/ruby/gem/README.txt +++ /dev/null @@ -1,23 +0,0 @@ -=MessagePack - -== DESCRIPTION: - -Binary-based efficient data interchange format. - - -== LICENSE: - -Copyright (C) 2008 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/ruby/gem/Rakefile b/ruby/gem/Rakefile index e469154..87635cb 100644 --- a/ruby/gem/Rakefile +++ b/ruby/gem/Rakefile @@ -1,4 +1,133 @@ -require 'config/requirements' -require 'config/hoe' # setup Hoe + all gem configuration - -Dir['tasks/**/*.rake'].each { |rake| load rake } \ No newline at end of file +require 'rubygems' +require 'rake' +require 'rake/clean' +require 'rake/testtask' +require 'rake/packagetask' +require 'rake/gempackagetask' +require 'rake/rdoctask' +require 'rake/contrib/rubyforgepublisher' +require 'rake/contrib/sshpublisher' +require 'fileutils' +include FileUtils + +NAME = "msgpack" +AUTHOR = "FURUHASHI Sadayuki" +EMAIL = "frsyuki _at_ users.sourceforge.jp" +DESCRIPTION = "Binary-based efficient data interchange format." +RUBYFORGE_PROJECT = "msgpack" +HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org" +BIN_FILES = %w( ) +VERS = "0.3.0" + +#REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil +REV = nil +CLEAN.include ['**/.*.sw?', '*.gem', '.config'] +RDOC_OPTS = [ + '--title', "#{NAME} documentation", + "--charset", "utf-8", + "--opname", "index.html", + "--line-numbers", + "--main", "README", + "--inline-source", +] + +task :default => [:test] +task :package => [:clean] + +Rake::TestTask.new("test") do |t| + t.libs << "test" + t.pattern = "test/**/*_test.rb" + t.verbose = true +end + +spec = Gem::Specification.new do |s| + s.name = NAME + s.version = VERS + s.platform = Gem::Platform::RUBY + s.has_rdoc = true + s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] + s.rdoc_options += RDOC_OPTS + ['--exclude', '^(examples|extras)/'] + s.summary = DESCRIPTION + s.description = DESCRIPTION + s.author = AUTHOR + s.email = EMAIL + s.homepage = HOMEPATH + s.executables = BIN_FILES + s.rubyforge_project = RUBYFORGE_PROJECT + s.bindir = "bin" + s.require_path = "ext" + s.autorequire = "" + s.test_files = Dir["test/test_*.rb"] + + #s.add_dependency('activesupport', '>=1.3.1') + #s.required_ruby_version = '>= 1.8.2' + + s.files = %w(README ChangeLog Rakefile) + + Dir.glob("{bin,doc,test,lib,templates,generator,extras,website,script}/**/*") + + Dir.glob("ext/**/*.{h,c,rb}") + + Dir.glob("examples/**/*.rb") + + Dir.glob("tools/*.rb") + + Dir.glob("msgpack/*.h") + + s.extensions = FileList["ext/**/extconf.rb"].to_a +end + +Rake::GemPackageTask.new(spec) do |p| + p.need_tar = true + p.gem_spec = spec +end + +task :install do + name = "#{NAME}-#{VERS}.gem" + sh %{rake package} + sh %{sudo gem install pkg/#{name}} +end + +task :uninstall => [:clean] do + sh %{sudo gem uninstall #{NAME}} +end + + +#Rake::RDocTask.new do |rdoc| +# rdoc.rdoc_dir = 'html' +# rdoc.options += RDOC_OPTS +# rdoc.template = "resh" +# #rdoc.template = "#{ENV['template']}.rb" if ENV['template'] +# if ENV['DOC_FILES'] +# rdoc.rdoc_files.include(ENV['DOC_FILES'].split(/,\s*/)) +# else +# rdoc.rdoc_files.include('README', 'ChangeLog') +# rdoc.rdoc_files.include('lib/**/*.rb') +# rdoc.rdoc_files.include('ext/**/*.c') +# end +#end + +desc "Publish to RubyForge" +task :rubyforge => [:rdoc, :package] do + require 'rubyforge' + Rake::RubyForgePublisher.new(RUBYFORGE_PROJECT, 'frsyuki').upload +end + +desc 'Package and upload the release to rubyforge.' +task :release => [:clean, :package] do |t| + v = ENV["VERSION"] or abort "Must supply VERSION=x.y.z" + abort "Versions don't match #{v} vs #{VERS}" unless v == VERS + pkg = "pkg/#{NAME}-#{VERS}" + + rf = RubyForge.new + puts "Logging in" + rf.login + + c = rf.userconfig +# c["release_notes"] = description if description +# c["release_changes"] = changes if changes + c["preformatted"] = true + + files = [ + "#{pkg}.tgz", + "#{pkg}.gem" + ].compact + + puts "Releasing #{NAME} v. #{VERS}" + rf.add_release RUBYFORGE_PROJECT, NAME, VERS, *files +end diff --git a/ruby/gem/config/hoe.rb b/ruby/gem/config/hoe.rb deleted file mode 100644 index 8500acf..0000000 --- a/ruby/gem/config/hoe.rb +++ /dev/null @@ -1,75 +0,0 @@ -require 'msgpack/version' - -AUTHOR = 'FURUHASHI Sadayuki' # can also be an array of Authors -EMAIL = "fr _at_ syuki.skr.jp" -DESCRIPTION = "An object-oriented parser generator based on Parser Expression Grammar" -GEM_NAME = 'msgpack' # what ppl will type to install your gem -RUBYFORGE_PROJECT = 'msgpack' # The unix name for your project -HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org" -DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}" -EXTRA_DEPENDENCIES = [ -# ['activesupport', '>= 1.3.1'] -] # An array of rubygem dependencies [name, version] - -@config_file = "~/.rubyforge/user-config.yml" -@config = nil -RUBYFORGE_USERNAME = "unknown" -def rubyforge_username - unless @config - begin - @config = YAML.load(File.read(File.expand_path(@config_file))) - rescue - puts <<-EOS -ERROR: No rubyforge config file found: #{@config_file} -Run 'rubyforge setup' to prepare your env for access to Rubyforge - - See http://newgem.rubyforge.org/rubyforge.html for more details - EOS - exit - end - end - RUBYFORGE_USERNAME.replace @config["username"] -end - - -REV = nil -# UNCOMMENT IF REQUIRED: -# REV = YAML.load(`svn info`)['Revision'] -VERS = MessagePack::VERSION::STRING + (REV ? ".#{REV}" : "") -RDOC_OPTS = ['--quiet', '--title', 'msgpack documentation', - "--opname", "index.html", - "--line-numbers", - "--main", "README", - "--inline-source"] - -class Hoe - def extra_deps - @extra_deps.reject! { |x| Array(x).first == 'hoe' } - @extra_deps - end -end - -# Generate all the Rake tasks -# Run 'rake -T' to see list of generated tasks (from gem root directory) -$hoe = Hoe.new(GEM_NAME, VERS) do |p| - p.developer(AUTHOR, EMAIL) - p.description = DESCRIPTION - p.summary = DESCRIPTION - p.url = HOMEPATH - p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT - p.test_globs = ["test/**/test_*.rb"] - p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store'] #An array of file patterns to delete on clean. - - # == Optional - p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n") - #p.extra_deps = EXTRA_DEPENDENCIES - - p.spec_extras = { # A hash of extra values to set in the gemspec. - :extensions => %w[ext/extconf.rb] - } -end - -CHANGES = $hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n") -PATH = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}" -$hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc') -$hoe.rsync_args = '-av --delete --ignore-errors' -$hoe.spec.post_install_message = File.open(File.dirname(__FILE__) + "/../PostInstall.txt").read rescue "" diff --git a/ruby/gem/config/requirements.rb b/ruby/gem/config/requirements.rb deleted file mode 100644 index 9292b69..0000000 --- a/ruby/gem/config/requirements.rb +++ /dev/null @@ -1,15 +0,0 @@ -require 'fileutils' -include FileUtils - -require 'rubygems' -%w[rake hoe newgem rubigen].each do |req_gem| - begin - require req_gem - rescue LoadError - puts "This Rakefile requires the '#{req_gem}' RubyGem." - puts "Installation: gem install #{req_gem} -y" - exit - end -end - -$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib])) diff --git a/ruby/gem/lib/msgpack/version.rb b/ruby/gem/lib/msgpack/version.rb deleted file mode 100644 index 433e6fc..0000000 --- a/ruby/gem/lib/msgpack/version.rb +++ /dev/null @@ -1,9 +0,0 @@ -module MessagePack - module VERSION #:nodoc: - MAJOR = 0 - MINOR = 3 - TINY = 0 - - STRING = [MAJOR, MINOR, TINY].join('.') - end -end diff --git a/ruby/gem/script/console b/ruby/gem/script/console deleted file mode 100755 index 76f32a0..0000000 --- a/ruby/gem/script/console +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby -# File: script/console -irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb' - -libs = " -r irb/completion" -# Perhaps use a console_lib to store any extra methods I may want available in the cosole -# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}" -libs << " -r #{File.dirname(__FILE__) + '/../lib/msgpack.rb'}" -puts "Loading msgpack gem" -exec "#{irb} #{libs} --simple-prompt" diff --git a/ruby/gem/script/destroy b/ruby/gem/script/destroy deleted file mode 100755 index e48464d..0000000 --- a/ruby/gem/script/destroy +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env ruby -APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..')) - -begin - require 'rubigen' -rescue LoadError - require 'rubygems' - require 'rubigen' -end -require 'rubigen/scripts/destroy' - -ARGV.shift if ['--help', '-h'].include?(ARGV[0]) -RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit] -RubiGen::Scripts::Destroy.new.run(ARGV) diff --git a/ruby/gem/script/generate b/ruby/gem/script/generate deleted file mode 100755 index c27f655..0000000 --- a/ruby/gem/script/generate +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env ruby -APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..')) - -begin - require 'rubigen' -rescue LoadError - require 'rubygems' - require 'rubigen' -end -require 'rubigen/scripts/generate' - -ARGV.shift if ['--help', '-h'].include?(ARGV[0]) -RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit] -RubiGen::Scripts::Generate.new.run(ARGV) diff --git a/ruby/gem/script/txt2html b/ruby/gem/script/txt2html deleted file mode 100755 index 09c583f..0000000 --- a/ruby/gem/script/txt2html +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env ruby - -GEM_NAME = 'msgpack' # what ppl will type to install your gem -RUBYFORGE_PROJECT = 'msgpack' - -require 'rubygems' -begin - require 'newgem' - require 'rubyforge' -rescue LoadError - puts "\n\nGenerating the website requires the newgem RubyGem" - puts "Install: gem install newgem\n\n" - exit(1) -end -require 'redcloth' -require 'syntax/convertors/html' -require 'erb' -require File.dirname(__FILE__) + "/../lib/#{GEM_NAME}/version.rb" - -version = MessagePack::VERSION::STRING -download = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}" - -def rubyforge_project_id - RubyForge.new.autoconfig["group_ids"][RUBYFORGE_PROJECT] -end - -class Fixnum - def ordinal - # teens - return 'th' if (10..19).include?(self % 100) - # others - case self % 10 - when 1: return 'st' - when 2: return 'nd' - when 3: return 'rd' - else return 'th' - end - end -end - -class Time - def pretty - return "#{mday}#{mday.ordinal} #{strftime('%B')} #{year}" - end -end - -def convert_syntax(syntax, source) - return Syntax::Convertors::HTML.for_syntax(syntax).convert(source).gsub(%r!^
|
$!,'') -end - -if ARGV.length >= 1 - src, template = ARGV - template ||= File.join(File.dirname(__FILE__), '/../website/template.html.erb') -else - puts("Usage: #{File.split($0).last} source.txt [template.html.erb] > output.html") - exit! -end - -template = ERB.new(File.open(template).read) - -title = nil -body = nil -File.open(src) do |fsrc| - title_text = fsrc.readline - body_text_template = fsrc.read - body_text = ERB.new(body_text_template).result(binding) - syntax_items = [] - body_text.gsub!(%r!<(pre|code)[^>]*?syntax=['"]([^'"]+)[^>]*>(.*?)!m){ - ident = syntax_items.length - element, syntax, source = $1, $2, $3 - syntax_items << "<#{element} class='syntax'>#{convert_syntax(syntax, source)}" - "syntax-temp-#{ident}" - } - title = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip - body = RedCloth.new(body_text).to_html - body.gsub!(%r!(?:
)?syntax-temp-(\d+)(?:
)?!){ syntax_items[$1.to_i] } -end -stat = File.stat(src) -created = stat.ctime -modified = stat.mtime - -$stdout << template.result(binding) diff --git a/ruby/gem/setup.rb b/ruby/gem/setup.rb deleted file mode 100644 index 424a5f3..0000000 --- a/ruby/gem/setup.rb +++ /dev/null @@ -1,1585 +0,0 @@ -# -# setup.rb -# -# Copyright (c) 2000-2005 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the terms of -# the GNU LGPL, Lesser General Public License version 2.1. -# - -unless Enumerable.method_defined?(:map) # Ruby 1.4.6 - module Enumerable - alias map collect - end -end - -unless File.respond_to?(:read) # Ruby 1.6 - def File.read(fname) - open(fname) {|f| - return f.read - } - end -end - -unless Errno.const_defined?(:ENOTEMPTY) # Windows? - module Errno - class ENOTEMPTY - # We do not raise this exception, implementation is not needed. - end - end -end - -def File.binread(fname) - open(fname, 'rb') {|f| - return f.read - } -end - -# for corrupted Windows' stat(2) -def File.dir?(path) - File.directory?((path[-1,1] == '/') ? path : path + '/') -end - - -class ConfigTable - - include Enumerable - - def initialize(rbconfig) - @rbconfig = rbconfig - @items = [] - @table = {} - # options - @install_prefix = nil - @config_opt = nil - @verbose = true - @no_harm = false - end - - attr_accessor :install_prefix - attr_accessor :config_opt - - attr_writer :verbose - - def verbose? - @verbose - end - - attr_writer :no_harm - - def no_harm? - @no_harm - end - - def [](key) - lookup(key).resolve(self) - end - - def []=(key, val) - lookup(key).set val - end - - def names - @items.map {|i| i.name } - end - - def each(&block) - @items.each(&block) - end - - def key?(name) - @table.key?(name) - end - - def lookup(name) - @table[name] or setup_rb_error "no such config item: #{name}" - end - - def add(item) - @items.push item - @table[item.name] = item - end - - def remove(name) - item = lookup(name) - @items.delete_if {|i| i.name == name } - @table.delete_if {|name, i| i.name == name } - item - end - - def load_script(path, inst = nil) - if File.file?(path) - MetaConfigEnvironment.new(self, inst).instance_eval File.read(path), path - end - end - - def savefile - '.config' - end - - def load_savefile - begin - File.foreach(savefile()) do |line| - k, v = *line.split(/=/, 2) - self[k] = v.strip - end - rescue Errno::ENOENT - setup_rb_error $!.message + "\n#{File.basename($0)} config first" - end - end - - def save - @items.each {|i| i.value } - File.open(savefile(), 'w') {|f| - @items.each do |i| - f.printf "%s=%s\n", i.name, i.value if i.value? and i.value - end - } - end - - def load_standard_entries - standard_entries(@rbconfig).each do |ent| - add ent - end - end - - def standard_entries(rbconfig) - c = rbconfig - - rubypath = File.join(c['bindir'], c['ruby_install_name'] + c['EXEEXT']) - - major = c['MAJOR'].to_i - minor = c['MINOR'].to_i - teeny = c['TEENY'].to_i - version = "#{major}.#{minor}" - - # ruby ver. >= 1.4.4? - newpath_p = ((major >= 2) or - ((major == 1) and - ((minor >= 5) or - ((minor == 4) and (teeny >= 4))))) - - if c['rubylibdir'] - # V > 1.6.3 - libruby = "#{c['prefix']}/lib/ruby" - librubyver = c['rubylibdir'] - librubyverarch = c['archdir'] - siteruby = c['sitedir'] - siterubyver = c['sitelibdir'] - siterubyverarch = c['sitearchdir'] - elsif newpath_p - # 1.4.4 <= V <= 1.6.3 - libruby = "#{c['prefix']}/lib/ruby" - librubyver = "#{c['prefix']}/lib/ruby/#{version}" - librubyverarch = "#{c['prefix']}/lib/ruby/#{version}/#{c['arch']}" - siteruby = c['sitedir'] - siterubyver = "$siteruby/#{version}" - siterubyverarch = "$siterubyver/#{c['arch']}" - else - # V < 1.4.4 - libruby = "#{c['prefix']}/lib/ruby" - librubyver = "#{c['prefix']}/lib/ruby/#{version}" - librubyverarch = "#{c['prefix']}/lib/ruby/#{version}/#{c['arch']}" - siteruby = "#{c['prefix']}/lib/ruby/#{version}/site_ruby" - siterubyver = siteruby - siterubyverarch = "$siterubyver/#{c['arch']}" - end - parameterize = lambda {|path| - path.sub(/\A#{Regexp.quote(c['prefix'])}/, '$prefix') - } - - if arg = c['configure_args'].split.detect {|arg| /--with-make-prog=/ =~ arg } - makeprog = arg.sub(/'/, '').split(/=/, 2)[1] - else - makeprog = 'make' - end - - [ - ExecItem.new('installdirs', 'std/site/home', - 'std: install under libruby; site: install under site_ruby; home: install under $HOME')\ - {|val, table| - case val - when 'std' - table['rbdir'] = '$librubyver' - table['sodir'] = '$librubyverarch' - when 'site' - table['rbdir'] = '$siterubyver' - table['sodir'] = '$siterubyverarch' - when 'home' - setup_rb_error '$HOME was not set' unless ENV['HOME'] - table['prefix'] = ENV['HOME'] - table['rbdir'] = '$libdir/ruby' - table['sodir'] = '$libdir/ruby' - end - }, - PathItem.new('prefix', 'path', c['prefix'], - 'path prefix of target environment'), - PathItem.new('bindir', 'path', parameterize.call(c['bindir']), - 'the directory for commands'), - PathItem.new('libdir', 'path', parameterize.call(c['libdir']), - 'the directory for libraries'), - PathItem.new('datadir', 'path', parameterize.call(c['datadir']), - 'the directory for shared data'), - PathItem.new('mandir', 'path', parameterize.call(c['mandir']), - 'the directory for man pages'), - PathItem.new('sysconfdir', 'path', parameterize.call(c['sysconfdir']), - 'the directory for system configuration files'), - PathItem.new('localstatedir', 'path', parameterize.call(c['localstatedir']), - 'the directory for local state data'), - PathItem.new('libruby', 'path', libruby, - 'the directory for ruby libraries'), - PathItem.new('librubyver', 'path', librubyver, - 'the directory for standard ruby libraries'), - PathItem.new('librubyverarch', 'path', librubyverarch, - 'the directory for standard ruby extensions'), - PathItem.new('siteruby', 'path', siteruby, - 'the directory for version-independent aux ruby libraries'), - PathItem.new('siterubyver', 'path', siterubyver, - 'the directory for aux ruby libraries'), - PathItem.new('siterubyverarch', 'path', siterubyverarch, - 'the directory for aux ruby binaries'), - PathItem.new('rbdir', 'path', '$siterubyver', - 'the directory for ruby scripts'), - PathItem.new('sodir', 'path', '$siterubyverarch', - 'the directory for ruby extentions'), - PathItem.new('rubypath', 'path', rubypath, - 'the path to set to #! line'), - ProgramItem.new('rubyprog', 'name', rubypath, - 'the ruby program using for installation'), - ProgramItem.new('makeprog', 'name', makeprog, - 'the make program to compile ruby extentions'), - SelectItem.new('shebang', 'all/ruby/never', 'ruby', - 'shebang line (#!) editing mode'), - BoolItem.new('without-ext', 'yes/no', 'no', - 'does not compile/install ruby extentions') - ] - end - private :standard_entries - - def load_multipackage_entries - multipackage_entries().each do |ent| - add ent - end - end - - def multipackage_entries - [ - PackageSelectionItem.new('with', 'name,name...', '', 'ALL', - 'package names that you want to install'), - PackageSelectionItem.new('without', 'name,name...', '', 'NONE', - 'package names that you do not want to install') - ] - end - private :multipackage_entries - - ALIASES = { - 'std-ruby' => 'librubyver', - 'stdruby' => 'librubyver', - 'rubylibdir' => 'librubyver', - 'archdir' => 'librubyverarch', - 'site-ruby-common' => 'siteruby', # For backward compatibility - 'site-ruby' => 'siterubyver', # For backward compatibility - 'bin-dir' => 'bindir', - 'bin-dir' => 'bindir', - 'rb-dir' => 'rbdir', - 'so-dir' => 'sodir', - 'data-dir' => 'datadir', - 'ruby-path' => 'rubypath', - 'ruby-prog' => 'rubyprog', - 'ruby' => 'rubyprog', - 'make-prog' => 'makeprog', - 'make' => 'makeprog' - } - - def fixup - ALIASES.each do |ali, name| - @table[ali] = @table[name] - end - @items.freeze - @table.freeze - @options_re = /\A--(#{@table.keys.join('|')})(?:=(.*))?\z/ - end - - def parse_opt(opt) - m = @options_re.match(opt) or setup_rb_error "config: unknown option #{opt}" - m.to_a[1,2] - end - - def dllext - @rbconfig['DLEXT'] - end - - def value_config?(name) - lookup(name).value? - end - - class Item - def initialize(name, template, default, desc) - @name = name.freeze - @template = template - @value = default - @default = default - @description = desc - end - - attr_reader :name - attr_reader :description - - attr_accessor :default - alias help_default default - - def help_opt - "--#{@name}=#{@template}" - end - - def value? - true - end - - def value - @value - end - - def resolve(table) - @value.gsub(%r<\$([^/]+)>) { table[$1] } - end - - def set(val) - @value = check(val) - end - - private - - def check(val) - setup_rb_error "config: --#{name} requires argument" unless val - val - end - end - - class BoolItem < Item - def config_type - 'bool' - end - - def help_opt - "--#{@name}" - end - - private - - def check(val) - return 'yes' unless val - case val - when /\Ay(es)?\z/i, /\At(rue)?\z/i then 'yes' - when /\An(o)?\z/i, /\Af(alse)\z/i then 'no' - else - setup_rb_error "config: --#{@name} accepts only yes/no for argument" - end - end - end - - class PathItem < Item - def config_type - 'path' - end - - private - - def check(path) - setup_rb_error "config: --#{@name} requires argument" unless path - path[0,1] == '$' ? path : File.expand_path(path) - end - end - - class ProgramItem < Item - def config_type - 'program' - end - end - - class SelectItem < Item - def initialize(name, selection, default, desc) - super - @ok = selection.split('/') - end - - def config_type - 'select' - end - - private - - def check(val) - unless @ok.include?(val.strip) - setup_rb_error "config: use --#{@name}=#{@template} (#{val})" - end - val.strip - end - end - - class ExecItem < Item - def initialize(name, selection, desc, &block) - super name, selection, nil, desc - @ok = selection.split('/') - @action = block - end - - def config_type - 'exec' - end - - def value? - false - end - - def resolve(table) - setup_rb_error "$#{name()} wrongly used as option value" - end - - undef set - - def evaluate(val, table) - v = val.strip.downcase - unless @ok.include?(v) - setup_rb_error "invalid option --#{@name}=#{val} (use #{@template})" - end - @action.call v, table - end - end - - class PackageSelectionItem < Item - def initialize(name, template, default, help_default, desc) - super name, template, default, desc - @help_default = help_default - end - - attr_reader :help_default - - def config_type - 'package' - end - - private - - def check(val) - unless File.dir?("packages/#{val}") - setup_rb_error "config: no such package: #{val}" - end - val - end - end - - class MetaConfigEnvironment - def initialize(config, installer) - @config = config - @installer = installer - end - - def config_names - @config.names - end - - def config?(name) - @config.key?(name) - end - - def bool_config?(name) - @config.lookup(name).config_type == 'bool' - end - - def path_config?(name) - @config.lookup(name).config_type == 'path' - end - - def value_config?(name) - @config.lookup(name).config_type != 'exec' - end - - def add_config(item) - @config.add item - end - - def add_bool_config(name, default, desc) - @config.add BoolItem.new(name, 'yes/no', default ? 'yes' : 'no', desc) - end - - def add_path_config(name, default, desc) - @config.add PathItem.new(name, 'path', default, desc) - end - - def set_config_default(name, default) - @config.lookup(name).default = default - end - - def remove_config(name) - @config.remove(name) - end - - # For only multipackage - def packages - raise '[setup.rb fatal] multi-package metaconfig API packages() called for single-package; contact application package vendor' unless @installer - @installer.packages - end - - # For only multipackage - def declare_packages(list) - raise '[setup.rb fatal] multi-package metaconfig API declare_packages() called for single-package; contact application package vendor' unless @installer - @installer.packages = list - end - end - -end # class ConfigTable - - -# This module requires: #verbose?, #no_harm? -module FileOperations - - def mkdir_p(dirname, prefix = nil) - dirname = prefix + File.expand_path(dirname) if prefix - $stderr.puts "mkdir -p #{dirname}" if verbose? - return if no_harm? - - # Does not check '/', it's too abnormal. - dirs = File.expand_path(dirname).split(%r<(?=/)>) - if /\A[a-z]:\z/i =~ dirs[0] - disk = dirs.shift - dirs[0] = disk + dirs[0] - end - dirs.each_index do |idx| - path = dirs[0..idx].join('') - Dir.mkdir path unless File.dir?(path) - end - end - - def rm_f(path) - $stderr.puts "rm -f #{path}" if verbose? - return if no_harm? - force_remove_file path - end - - def rm_rf(path) - $stderr.puts "rm -rf #{path}" if verbose? - return if no_harm? - remove_tree path - end - - def remove_tree(path) - if File.symlink?(path) - remove_file path - elsif File.dir?(path) - remove_tree0 path - else - force_remove_file path - end - end - - def remove_tree0(path) - Dir.foreach(path) do |ent| - next if ent == '.' - next if ent == '..' - entpath = "#{path}/#{ent}" - if File.symlink?(entpath) - remove_file entpath - elsif File.dir?(entpath) - remove_tree0 entpath - else - force_remove_file entpath - end - end - begin - Dir.rmdir path - rescue Errno::ENOTEMPTY - # directory may not be empty - end - end - - def move_file(src, dest) - force_remove_file dest - begin - File.rename src, dest - rescue - File.open(dest, 'wb') {|f| - f.write File.binread(src) - } - File.chmod File.stat(src).mode, dest - File.unlink src - end - end - - def force_remove_file(path) - begin - remove_file path - rescue - end - end - - def remove_file(path) - File.chmod 0777, path - File.unlink path - end - - def install(from, dest, mode, prefix = nil) - $stderr.puts "install #{from} #{dest}" if verbose? - return if no_harm? - - realdest = prefix ? prefix + File.expand_path(dest) : dest - realdest = File.join(realdest, File.basename(from)) if File.dir?(realdest) - str = File.binread(from) - if diff?(str, realdest) - verbose_off { - rm_f realdest if File.exist?(realdest) - } - File.open(realdest, 'wb') {|f| - f.write str - } - File.chmod mode, realdest - - File.open("#{objdir_root()}/InstalledFiles", 'a') {|f| - if prefix - f.puts realdest.sub(prefix, '') - else - f.puts realdest - end - } - end - end - - def diff?(new_content, path) - return true unless File.exist?(path) - new_content != File.binread(path) - end - - def command(*args) - $stderr.puts args.join(' ') if verbose? - system(*args) or raise RuntimeError, - "system(#{args.map{|a| a.inspect }.join(' ')}) failed" - end - - def ruby(*args) - command config('rubyprog'), *args - end - - def make(task = nil) - command(*[config('makeprog'), task].compact) - end - - def extdir?(dir) - File.exist?("#{dir}/MANIFEST") or File.exist?("#{dir}/extconf.rb") - end - - def files_of(dir) - Dir.open(dir) {|d| - return d.select {|ent| File.file?("#{dir}/#{ent}") } - } - end - - DIR_REJECT = %w( . .. CVS SCCS RCS CVS.adm .svn ) - - def directories_of(dir) - Dir.open(dir) {|d| - return d.select {|ent| File.dir?("#{dir}/#{ent}") } - DIR_REJECT - } - end - -end - - -# This module requires: #srcdir_root, #objdir_root, #relpath -module HookScriptAPI - - def get_config(key) - @config[key] - end - - alias config get_config - - # obsolete: use metaconfig to change configuration - def set_config(key, val) - @config[key] = val - end - - # - # srcdir/objdir (works only in the package directory) - # - - def curr_srcdir - "#{srcdir_root()}/#{relpath()}" - end - - def curr_objdir - "#{objdir_root()}/#{relpath()}" - end - - def srcfile(path) - "#{curr_srcdir()}/#{path}" - end - - def srcexist?(path) - File.exist?(srcfile(path)) - end - - def srcdirectory?(path) - File.dir?(srcfile(path)) - end - - def srcfile?(path) - File.file?(srcfile(path)) - end - - def srcentries(path = '.') - Dir.open("#{curr_srcdir()}/#{path}") {|d| - return d.to_a - %w(. ..) - } - end - - def srcfiles(path = '.') - srcentries(path).select {|fname| - File.file?(File.join(curr_srcdir(), path, fname)) - } - end - - def srcdirectories(path = '.') - srcentries(path).select {|fname| - File.dir?(File.join(curr_srcdir(), path, fname)) - } - end - -end - - -class ToplevelInstaller - - Version = '3.4.1' - Copyright = 'Copyright (c) 2000-2005 Minero Aoki' - - TASKS = [ - [ 'all', 'do config, setup, then install' ], - [ 'config', 'saves your configurations' ], - [ 'show', 'shows current configuration' ], - [ 'setup', 'compiles ruby extentions and others' ], - [ 'install', 'installs files' ], - [ 'test', 'run all tests in test/' ], - [ 'clean', "does `make clean' for each extention" ], - [ 'distclean',"does `make distclean' for each extention" ] - ] - - def ToplevelInstaller.invoke - config = ConfigTable.new(load_rbconfig()) - config.load_standard_entries - config.load_multipackage_entries if multipackage? - config.fixup - klass = (multipackage?() ? ToplevelInstallerMulti : ToplevelInstaller) - klass.new(File.dirname($0), config).invoke - end - - def ToplevelInstaller.multipackage? - File.dir?(File.dirname($0) + '/packages') - end - - def ToplevelInstaller.load_rbconfig - if arg = ARGV.detect {|arg| /\A--rbconfig=/ =~ arg } - ARGV.delete(arg) - load File.expand_path(arg.split(/=/, 2)[1]) - $".push 'rbconfig.rb' - else - require 'rbconfig' - end - ::Config::CONFIG - end - - def initialize(ardir_root, config) - @ardir = File.expand_path(ardir_root) - @config = config - # cache - @valid_task_re = nil - end - - def config(key) - @config[key] - end - - def inspect - "#<#{self.class} #{__id__()}>" - end - - def invoke - run_metaconfigs - case task = parsearg_global() - when nil, 'all' - parsearg_config - init_installers - exec_config - exec_setup - exec_install - else - case task - when 'config', 'test' - ; - when 'clean', 'distclean' - @config.load_savefile if File.exist?(@config.savefile) - else - @config.load_savefile - end - __send__ "parsearg_#{task}" - init_installers - __send__ "exec_#{task}" - end - end - - def run_metaconfigs - @config.load_script "#{@ardir}/metaconfig" - end - - def init_installers - @installer = Installer.new(@config, @ardir, File.expand_path('.')) - end - - # - # Hook Script API bases - # - - def srcdir_root - @ardir - end - - def objdir_root - '.' - end - - def relpath - '.' - end - - # - # Option Parsing - # - - def parsearg_global - while arg = ARGV.shift - case arg - when /\A\w+\z/ - setup_rb_error "invalid task: #{arg}" unless valid_task?(arg) - return arg - when '-q', '--quiet' - @config.verbose = false - when '--verbose' - @config.verbose = true - when '--help' - print_usage $stdout - exit 0 - when '--version' - puts "#{File.basename($0)} version #{Version}" - exit 0 - when '--copyright' - puts Copyright - exit 0 - else - setup_rb_error "unknown global option '#{arg}'" - end - end - nil - end - - def valid_task?(t) - valid_task_re() =~ t - end - - def valid_task_re - @valid_task_re ||= /\A(?:#{TASKS.map {|task,desc| task }.join('|')})\z/ - end - - def parsearg_no_options - unless ARGV.empty? - task = caller(0).first.slice(%r<`parsearg_(\w+)'>, 1) - setup_rb_error "#{task}: unknown options: #{ARGV.join(' ')}" - end - end - - alias parsearg_show parsearg_no_options - alias parsearg_setup parsearg_no_options - alias parsearg_test parsearg_no_options - alias parsearg_clean parsearg_no_options - alias parsearg_distclean parsearg_no_options - - def parsearg_config - evalopt = [] - set = [] - @config.config_opt = [] - while i = ARGV.shift - if /\A--?\z/ =~ i - @config.config_opt = ARGV.dup - break - end - name, value = *@config.parse_opt(i) - if @config.value_config?(name) - @config[name] = value - else - evalopt.push [name, value] - end - set.push name - end - evalopt.each do |name, value| - @config.lookup(name).evaluate value, @config - end - # Check if configuration is valid - set.each do |n| - @config[n] if @config.value_config?(n) - end - end - - def parsearg_install - @config.no_harm = false - @config.install_prefix = '' - while a = ARGV.shift - case a - when '--no-harm' - @config.no_harm = true - when /\A--prefix=/ - path = a.split(/=/, 2)[1] - path = File.expand_path(path) unless path[0,1] == '/' - @config.install_prefix = path - else - setup_rb_error "install: unknown option #{a}" - end - end - end - - def print_usage(out) - out.puts 'Typical Installation Procedure:' - out.puts " $ ruby #{File.basename $0} config" - out.puts " $ ruby #{File.basename $0} setup" - out.puts " # ruby #{File.basename $0} install (may require root privilege)" - out.puts - out.puts 'Detailed Usage:' - out.puts " ruby #{File.basename $0} " - out.puts " ruby #{File.basename $0} [] []" - - fmt = " %-24s %s\n" - out.puts - out.puts 'Global options:' - out.printf fmt, '-q,--quiet', 'suppress message outputs' - out.printf fmt, ' --verbose', 'output messages verbosely' - out.printf fmt, ' --help', 'print this message' - out.printf fmt, ' --version', 'print version and quit' - out.printf fmt, ' --copyright', 'print copyright and quit' - out.puts - out.puts 'Tasks:' - TASKS.each do |name, desc| - out.printf fmt, name, desc - end - - fmt = " %-24s %s [%s]\n" - out.puts - out.puts 'Options for CONFIG or ALL:' - @config.each do |item| - out.printf fmt, item.help_opt, item.description, item.help_default - end - out.printf fmt, '--rbconfig=path', 'rbconfig.rb to load',"running ruby's" - out.puts - out.puts 'Options for INSTALL:' - out.printf fmt, '--no-harm', 'only display what to do if given', 'off' - out.printf fmt, '--prefix=path', 'install path prefix', '' - out.puts - end - - # - # Task Handlers - # - - def exec_config - @installer.exec_config - @config.save # must be final - end - - def exec_setup - @installer.exec_setup - end - - def exec_install - @installer.exec_install - end - - def exec_test - @installer.exec_test - end - - def exec_show - @config.each do |i| - printf "%-20s %s\n", i.name, i.value if i.value? - end - end - - def exec_clean - @installer.exec_clean - end - - def exec_distclean - @installer.exec_distclean - end - -end # class ToplevelInstaller - - -class ToplevelInstallerMulti < ToplevelInstaller - - include FileOperations - - def initialize(ardir_root, config) - super - @packages = directories_of("#{@ardir}/packages") - raise 'no package exists' if @packages.empty? - @root_installer = Installer.new(@config, @ardir, File.expand_path('.')) - end - - def run_metaconfigs - @config.load_script "#{@ardir}/metaconfig", self - @packages.each do |name| - @config.load_script "#{@ardir}/packages/#{name}/metaconfig" - end - end - - attr_reader :packages - - def packages=(list) - raise 'package list is empty' if list.empty? - list.each do |name| - raise "directory packages/#{name} does not exist"\ - unless File.dir?("#{@ardir}/packages/#{name}") - end - @packages = list - end - - def init_installers - @installers = {} - @packages.each do |pack| - @installers[pack] = Installer.new(@config, - "#{@ardir}/packages/#{pack}", - "packages/#{pack}") - end - with = extract_selection(config('with')) - without = extract_selection(config('without')) - @selected = @installers.keys.select {|name| - (with.empty? or with.include?(name)) \ - and not without.include?(name) - } - end - - def extract_selection(list) - a = list.split(/,/) - a.each do |name| - setup_rb_error "no such package: #{name}" unless @installers.key?(name) - end - a - end - - def print_usage(f) - super - f.puts 'Inluded packages:' - f.puts ' ' + @packages.sort.join(' ') - f.puts - end - - # - # Task Handlers - # - - def exec_config - run_hook 'pre-config' - each_selected_installers {|inst| inst.exec_config } - run_hook 'post-config' - @config.save # must be final - end - - def exec_setup - run_hook 'pre-setup' - each_selected_installers {|inst| inst.exec_setup } - run_hook 'post-setup' - end - - def exec_install - run_hook 'pre-install' - each_selected_installers {|inst| inst.exec_install } - run_hook 'post-install' - end - - def exec_test - run_hook 'pre-test' - each_selected_installers {|inst| inst.exec_test } - run_hook 'post-test' - end - - def exec_clean - rm_f @config.savefile - run_hook 'pre-clean' - each_selected_installers {|inst| inst.exec_clean } - run_hook 'post-clean' - end - - def exec_distclean - rm_f @config.savefile - run_hook 'pre-distclean' - each_selected_installers {|inst| inst.exec_distclean } - run_hook 'post-distclean' - end - - # - # lib - # - - def each_selected_installers - Dir.mkdir 'packages' unless File.dir?('packages') - @selected.each do |pack| - $stderr.puts "Processing the package `#{pack}' ..." if verbose? - Dir.mkdir "packages/#{pack}" unless File.dir?("packages/#{pack}") - Dir.chdir "packages/#{pack}" - yield @installers[pack] - Dir.chdir '../..' - end - end - - def run_hook(id) - @root_installer.run_hook id - end - - # module FileOperations requires this - def verbose? - @config.verbose? - end - - # module FileOperations requires this - def no_harm? - @config.no_harm? - end - -end # class ToplevelInstallerMulti - - -class Installer - - FILETYPES = %w( bin lib ext data conf man ) - - include FileOperations - include HookScriptAPI - - def initialize(config, srcroot, objroot) - @config = config - @srcdir = File.expand_path(srcroot) - @objdir = File.expand_path(objroot) - @currdir = '.' - end - - def inspect - "#<#{self.class} #{File.basename(@srcdir)}>" - end - - def noop(rel) - end - - # - # Hook Script API base methods - # - - def srcdir_root - @srcdir - end - - def objdir_root - @objdir - end - - def relpath - @currdir - end - - # - # Config Access - # - - # module FileOperations requires this - def verbose? - @config.verbose? - end - - # module FileOperations requires this - def no_harm? - @config.no_harm? - end - - def verbose_off - begin - save, @config.verbose = @config.verbose?, false - yield - ensure - @config.verbose = save - end - end - - # - # TASK config - # - - def exec_config - exec_task_traverse 'config' - end - - alias config_dir_bin noop - alias config_dir_lib noop - - def config_dir_ext(rel) - extconf if extdir?(curr_srcdir()) - end - - alias config_dir_data noop - alias config_dir_conf noop - alias config_dir_man noop - - def extconf - ruby "#{curr_srcdir()}/extconf.rb", *@config.config_opt - end - - # - # TASK setup - # - - def exec_setup - exec_task_traverse 'setup' - end - - def setup_dir_bin(rel) - files_of(curr_srcdir()).each do |fname| - update_shebang_line "#{curr_srcdir()}/#{fname}" - end - end - - alias setup_dir_lib noop - - def setup_dir_ext(rel) - make if extdir?(curr_srcdir()) - end - - alias setup_dir_data noop - alias setup_dir_conf noop - alias setup_dir_man noop - - def update_shebang_line(path) - return if no_harm? - return if config('shebang') == 'never' - old = Shebang.load(path) - if old - $stderr.puts "warning: #{path}: Shebang line includes too many args. It is not portable and your program may not work." if old.args.size > 1 - new = new_shebang(old) - return if new.to_s == old.to_s - else - return unless config('shebang') == 'all' - new = Shebang.new(config('rubypath')) - end - $stderr.puts "updating shebang: #{File.basename(path)}" if verbose? - open_atomic_writer(path) {|output| - File.open(path, 'rb') {|f| - f.gets if old # discard - output.puts new.to_s - output.print f.read - } - } - end - - def new_shebang(old) - if /\Aruby/ =~ File.basename(old.cmd) - Shebang.new(config('rubypath'), old.args) - elsif File.basename(old.cmd) == 'env' and old.args.first == 'ruby' - Shebang.new(config('rubypath'), old.args[1..-1]) - else - return old unless config('shebang') == 'all' - Shebang.new(config('rubypath')) - end - end - - def open_atomic_writer(path, &block) - tmpfile = File.basename(path) + '.tmp' - begin - File.open(tmpfile, 'wb', &block) - File.rename tmpfile, File.basename(path) - ensure - File.unlink tmpfile if File.exist?(tmpfile) - end - end - - class Shebang - def Shebang.load(path) - line = nil - File.open(path) {|f| - line = f.gets - } - return nil unless /\A#!/ =~ line - parse(line) - end - - def Shebang.parse(line) - cmd, *args = *line.strip.sub(/\A\#!/, '').split(' ') - new(cmd, args) - end - - def initialize(cmd, args = []) - @cmd = cmd - @args = args - end - - attr_reader :cmd - attr_reader :args - - def to_s - "#! #{@cmd}" + (@args.empty? ? '' : " #{@args.join(' ')}") - end - end - - # - # TASK install - # - - def exec_install - rm_f 'InstalledFiles' - exec_task_traverse 'install' - end - - def install_dir_bin(rel) - install_files targetfiles(), "#{config('bindir')}/#{rel}", 0755 - end - - def install_dir_lib(rel) - install_files libfiles(), "#{config('rbdir')}/#{rel}", 0644 - end - - def install_dir_ext(rel) - return unless extdir?(curr_srcdir()) - install_files rubyextentions('.'), - "#{config('sodir')}/#{File.dirname(rel)}", - 0555 - end - - def install_dir_data(rel) - install_files targetfiles(), "#{config('datadir')}/#{rel}", 0644 - end - - def install_dir_conf(rel) - # FIXME: should not remove current config files - # (rename previous file to .old/.org) - install_files targetfiles(), "#{config('sysconfdir')}/#{rel}", 0644 - end - - def install_dir_man(rel) - install_files targetfiles(), "#{config('mandir')}/#{rel}", 0644 - end - - def install_files(list, dest, mode) - mkdir_p dest, @config.install_prefix - list.each do |fname| - install fname, dest, mode, @config.install_prefix - end - end - - def libfiles - glob_reject(%w(*.y *.output), targetfiles()) - end - - def rubyextentions(dir) - ents = glob_select("*.#{@config.dllext}", targetfiles()) - if ents.empty? - setup_rb_error "no ruby extention exists: 'ruby #{$0} setup' first" - end - ents - end - - def targetfiles - mapdir(existfiles() - hookfiles()) - end - - def mapdir(ents) - ents.map {|ent| - if File.exist?(ent) - then ent # objdir - else "#{curr_srcdir()}/#{ent}" # srcdir - end - } - end - - # picked up many entries from cvs-1.11.1/src/ignore.c - JUNK_FILES = %w( - core RCSLOG tags TAGS .make.state - .nse_depinfo #* .#* cvslog.* ,* .del-* *.olb - *~ *.old *.bak *.BAK *.orig *.rej _$* *$ - - *.org *.in .* - ) - - def existfiles - glob_reject(JUNK_FILES, (files_of(curr_srcdir()) | files_of('.'))) - end - - def hookfiles - %w( pre-%s post-%s pre-%s.rb post-%s.rb ).map {|fmt| - %w( config setup install clean ).map {|t| sprintf(fmt, t) } - }.flatten - end - - def glob_select(pat, ents) - re = globs2re([pat]) - ents.select {|ent| re =~ ent } - end - - def glob_reject(pats, ents) - re = globs2re(pats) - ents.reject {|ent| re =~ ent } - end - - GLOB2REGEX = { - '.' => '\.', - '$' => '\$', - '#' => '\#', - '*' => '.*' - } - - def globs2re(pats) - /\A(?:#{ - pats.map {|pat| pat.gsub(/[\.\$\#\*]/) {|ch| GLOB2REGEX[ch] } }.join('|') - })\z/ - end - - # - # TASK test - # - - TESTDIR = 'test' - - def exec_test - unless File.directory?('test') - $stderr.puts 'no test in this package' if verbose? - return - end - $stderr.puts 'Running tests...' if verbose? - begin - require 'test/unit' - rescue LoadError - setup_rb_error 'test/unit cannot loaded. You need Ruby 1.8 or later to invoke this task.' - end - runner = Test::Unit::AutoRunner.new(true) - runner.to_run << TESTDIR - runner.run - end - - # - # TASK clean - # - - def exec_clean - exec_task_traverse 'clean' - rm_f @config.savefile - rm_f 'InstalledFiles' - end - - alias clean_dir_bin noop - alias clean_dir_lib noop - alias clean_dir_data noop - alias clean_dir_conf noop - alias clean_dir_man noop - - def clean_dir_ext(rel) - return unless extdir?(curr_srcdir()) - make 'clean' if File.file?('Makefile') - end - - # - # TASK distclean - # - - def exec_distclean - exec_task_traverse 'distclean' - rm_f @config.savefile - rm_f 'InstalledFiles' - end - - alias distclean_dir_bin noop - alias distclean_dir_lib noop - - def distclean_dir_ext(rel) - return unless extdir?(curr_srcdir()) - make 'distclean' if File.file?('Makefile') - end - - alias distclean_dir_data noop - alias distclean_dir_conf noop - alias distclean_dir_man noop - - # - # Traversing - # - - def exec_task_traverse(task) - run_hook "pre-#{task}" - FILETYPES.each do |type| - if type == 'ext' and config('without-ext') == 'yes' - $stderr.puts 'skipping ext/* by user option' if verbose? - next - end - traverse task, type, "#{task}_dir_#{type}" - end - run_hook "post-#{task}" - end - - def traverse(task, rel, mid) - dive_into(rel) { - run_hook "pre-#{task}" - __send__ mid, rel.sub(%r[\A.*?(?:/|\z)], '') - directories_of(curr_srcdir()).each do |d| - traverse task, "#{rel}/#{d}", mid - end - run_hook "post-#{task}" - } - end - - def dive_into(rel) - return unless File.dir?("#{@srcdir}/#{rel}") - - dir = File.basename(rel) - Dir.mkdir dir unless File.dir?(dir) - prevdir = Dir.pwd - Dir.chdir dir - $stderr.puts '---> ' + rel if verbose? - @currdir = rel - yield - Dir.chdir prevdir - $stderr.puts '<--- ' + rel if verbose? - @currdir = File.dirname(rel) - end - - def run_hook(id) - path = [ "#{curr_srcdir()}/#{id}", - "#{curr_srcdir()}/#{id}.rb" ].detect {|cand| File.file?(cand) } - return unless path - begin - instance_eval File.read(path), path, 1 - rescue - raise if $DEBUG - setup_rb_error "hook #{path} failed:\n" + $!.message - end - end - -end # class Installer - - -class SetupError < StandardError; end - -def setup_rb_error(msg) - raise SetupError, msg -end - -if $0 == __FILE__ - begin - ToplevelInstaller.invoke - rescue SetupError - raise if $DEBUG - $stderr.puts $!.message - $stderr.puts "Try 'ruby #{$0} --help' for detailed usage." - exit 1 - end -end diff --git a/ruby/gem/tasks/deployment.rake b/ruby/gem/tasks/deployment.rake deleted file mode 100644 index 2f43742..0000000 --- a/ruby/gem/tasks/deployment.rake +++ /dev/null @@ -1,34 +0,0 @@ -desc 'Release the website and new gem version' -task :deploy => [:check_version, :website, :release] do - puts "Remember to create SVN tag:" - puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " + - "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} " - puts "Suggested comment:" - puts "Tagging release #{CHANGES}" -end - -desc 'Runs tasks website_generate and install_gem as a local deployment of the gem' -task :local_deploy => [:website_generate, :install_gem] - -task :check_version do - unless ENV['VERSION'] - puts 'Must pass a VERSION=x.y.z release version' - exit - end - unless ENV['VERSION'] == VERS - puts "Please update your version.rb to match the release version, currently #{VERS}" - exit - end -end - -desc 'Install the package as a gem, without generating documentation(ri/rdoc)' -task :install_gem_no_doc => [:clean, :package] do - sh "#{'sudo ' unless Hoe::WINDOZE }gem install pkg/*.gem --no-rdoc --no-ri" -end - -namespace :manifest do - desc 'Recreate Manifest.txt to include ALL files' - task :refresh do - `rake check_manifest | patch -p0 > Manifest.txt` - end -end \ No newline at end of file diff --git a/ruby/gem/tasks/environment.rake b/ruby/gem/tasks/environment.rake deleted file mode 100644 index 691ed3b..0000000 --- a/ruby/gem/tasks/environment.rake +++ /dev/null @@ -1,7 +0,0 @@ -task :ruby_env do - RUBY_APP = if RUBY_PLATFORM =~ /java/ - "jruby" - else - "ruby" - end unless defined? RUBY_APP -end diff --git a/ruby/gem/test/test_helper.rb b/ruby/gem/test/test_helper.rb new file mode 100644 index 0000000..3088faa --- /dev/null +++ b/ruby/gem/test/test_helper.rb @@ -0,0 +1,3 @@ +require 'test/unit' +require File.dirname(__FILE__) + '/../ext/msgpack' + diff --git a/ruby/gengem.sh b/ruby/gengem.sh index de9f4a5..be9d14b 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -6,6 +6,7 @@ cp pack.h gem/ext/ cp rbinit.c gem/ext/ cp unpack.c gem/ext/ cp unpack.h gem/ext/ +cat test_case.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FILE__) + '\/test_helper.rb'/" > gem/test/msgpack_test.rb #cp ../README gem/README.txt cp ../msgpack/pack_define.h gem/msgpack/ cp ../msgpack/pack_template.h gem/msgpack/ diff --git a/ruby/test_case.rb b/ruby/test_case.rb index 1e2d474..1a06b10 100644 --- a/ruby/test_case.rb +++ b/ruby/test_case.rb @@ -1,5 +1,6 @@ -require 'test/unit' +#!/usr/bin/env ruby require 'msgpack' +require 'test/unit' class MessagePackTestFormat < Test::Unit::TestCase def self.it(name, &block) From 761b3980834e4644ce0ce57f6db13d1c93883380 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 18:39:30 +0900 Subject: [PATCH 0056/1172] rewrite msgpack::object::via --- c/object.c | 24 ++++++++--------- c/object.h | 32 +++++++++++++++++------ c/pack.h | 58 ++++++++++++++++++++++-------------------- c/zone.c | 2 +- cpp/object.cpp | 48 ++++++++++++++++------------------ cpp/object.hpp | 56 ++++++++++++++++++++++++++-------------- cpp/type/array.hpp | 6 ++--- cpp/type/map.hpp | 36 +++++++++++++------------- cpp/type/raw.hpp | 10 ++++---- cpp/type/tuple.hpp.erb | 4 +-- cpp/unpack.cpp | 22 ++++++++-------- 11 files changed, 165 insertions(+), 133 deletions(-) diff --git a/c/object.c b/c/object.c index 7a41064..6376438 100644 --- a/c/object.c +++ b/c/object.c @@ -98,42 +98,42 @@ static inline msgpack_object msgpack_unpack_array(unpack_user* u, unsigned int n { msgpack_object o; o.type = MSGPACK_OBJECT_ARRAY; - o.via.container.size = 0; - o.via.container.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); - if(o.via.container.ptr == NULL) { u->failed = true; } + o.via.array.size = 0; + o.via.array.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); + if(o.via.array.ptr == NULL) { u->failed = true; } return o; } static inline void msgpack_unpack_array_item(unpack_user* u, msgpack_object* c, msgpack_object o) { if(u->failed) { return; } - c->via.container.ptr[ c->via.container.size++ ] = o; + c->via.array.ptr[ c->via.array.size++ ] = o; } static inline msgpack_object msgpack_unpack_map(unpack_user* u, unsigned int n) { msgpack_object o; o.type = MSGPACK_OBJECT_MAP; - o.via.container.size = 0; - o.via.container.ptr = msgpack_zone_malloc(u->z, n*2*sizeof(msgpack_object)); - if(o.via.container.ptr == NULL) { u->failed = true; } + o.via.map.size = 0; + o.via.map.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object_kv)); + if(o.via.map.ptr == NULL) { u->failed = true; } return o; } static inline void msgpack_unpack_map_item(unpack_user* u, msgpack_object* c, msgpack_object k, msgpack_object v) { if(u->failed) { return; } - c->via.container.ptr[ c->via.container.size ] = k; - c->via.container.ptr[ c->via.container.size+1 ] = v; - ++c->via.container.size; + c->via.map.ptr[c->via.map.size].key = k; + c->via.map.ptr[c->via.map.size].val = v; + ++c->via.map.size; } static inline msgpack_object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) { msgpack_object o; o.type = MSGPACK_OBJECT_RAW; - o.via.ref.ptr = p; - o.via.ref.size = l; + o.via.raw.ptr = p; + o.via.raw.size = l; u->referenced = true; return o; } diff --git a/c/object.h b/c/object.h index 1b60c57..b590ebe 100644 --- a/c/object.h +++ b/c/object.h @@ -41,19 +41,31 @@ typedef enum { struct _msgpack_object; +struct _msgpack_object_kv; + +typedef struct { + uint32_t size; + struct _msgpack_object* ptr; +} msgpack_object_array; + +typedef struct { + uint32_t size; + struct _msgpack_object_kv* ptr; +} msgpack_object_map; + +typedef struct { + uint32_t size; + const char* ptr; +} msgpack_object_raw; + typedef union { bool boolean; uint64_t u64; int64_t i64; double dec; - struct { - struct _msgpack_object* ptr; - uint32_t size; - } container; - struct { - const char* ptr; - uint32_t size; - } ref; + msgpack_object_array array; + msgpack_object_map map; + msgpack_object_raw raw; } msgpack_object_union; typedef struct _msgpack_object { @@ -61,6 +73,10 @@ typedef struct _msgpack_object { msgpack_object_union via; } msgpack_object; +typedef struct _msgpack_object_kv { + msgpack_object key; + msgpack_object val; +} msgpack_object_kv; typedef enum { MSGPACK_OBJECT_PARSE_SUCCESS = 0, diff --git a/c/pack.h b/c/pack.h index 752cbc4..a510eff 100644 --- a/c/pack.h +++ b/c/pack.h @@ -21,6 +21,7 @@ #include #include #include +#include "msgpack/pack_define.h" #ifdef __cplusplus extern "C" { @@ -34,42 +35,43 @@ typedef struct { msgpack_pack_write_t callback; } msgpack_pack_t; -void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback); +static void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback); -msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback); -void msgpack_pack_free(msgpack_pack_t* ctx); +static msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback); -int msgpack_pack_short(msgpack_pack_t* ctx, short d); -int msgpack_pack_int(msgpack_pack_t* ctx, int d); -int msgpack_pack_long(msgpack_pack_t* ctx, long d); -int msgpack_pack_long_long(msgpack_pack_t* ctx, long long d); -int msgpack_pack_unsigned_short(msgpack_pack_t* ctx, unsigned short d); -int msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); -int msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); -int msgpack_pack_unsigned_long_long(msgpack_pack_t* ctx, unsigned long long d); +static void msgpack_pack_free(msgpack_pack_t* ctx); -int msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); -int msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); -int msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); -int msgpack_pack_uint64(msgpack_pack_t* ctx, uint64_t d); -int msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); -int msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); -int msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); -int msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); +static int msgpack_pack_short(msgpack_pack_t* ctx, short d); +static int msgpack_pack_int(msgpack_pack_t* ctx, int d); +static int msgpack_pack_long(msgpack_pack_t* ctx, long d); +static int msgpack_pack_long_long(msgpack_pack_t* ctx, long long d); +static int msgpack_pack_unsigned_short(msgpack_pack_t* ctx, unsigned short d); +static int msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); +static int msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); +static int msgpack_pack_unsigned_long_long(msgpack_pack_t* ctx, unsigned long long d); -int msgpack_pack_float(msgpack_pack_t* ctx, float d); -int msgpack_pack_double(msgpack_pack_t* ctx, double d); +static int msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); +static int msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); +static int msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); +static int msgpack_pack_uint64(msgpack_pack_t* ctx, uint64_t d); +static int msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); +static int msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); +static int msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); +static int msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); -int msgpack_pack_nil(msgpack_pack_t* ctx); -int msgpack_pack_true(msgpack_pack_t* ctx); -int msgpack_pack_false(msgpack_pack_t* ctx); +static int msgpack_pack_float(msgpack_pack_t* ctx, float d); +static int msgpack_pack_double(msgpack_pack_t* ctx, double d); -int msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); +static int msgpack_pack_nil(msgpack_pack_t* ctx); +static int msgpack_pack_true(msgpack_pack_t* ctx); +static int msgpack_pack_false(msgpack_pack_t* ctx); -int msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); +static int msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); -int msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); -int msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); +static int msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); + +static int msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); +static int msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); diff --git a/c/zone.c b/c/zone.c index a41bb44..ccd702d 100644 --- a/c/zone.c +++ b/c/zone.c @@ -41,7 +41,7 @@ void msgpack_zone_free(msgpack_zone* z) if(z->array) { size_t i; for(i=0; i <= z->ntail; ++i) { - free(z->array[i].ptr); + free(z->array[i].alloc); } } free(z); diff --git a/cpp/object.cpp b/cpp/object.cpp index d217c06..eb45f77 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -46,16 +46,16 @@ std::ostream& operator<< (std::ostream& s, const object o) break; case type::RAW: - (s << '"').write(o.via.ref.ptr, o.via.ref.size) << '"'; + (s << '"').write(o.via.raw.ptr, o.via.raw.size) << '"'; break; case type::ARRAY: s << "["; - if(o.via.container.size != 0) { - object* p(o.via.container.ptr); + if(o.via.array.size != 0) { + object* p(o.via.array.ptr); s << *p; ++p; - for(object* const pend(o.via.container.ptr + o.via.container.size); + for(object* const pend(o.via.array.ptr + o.via.array.size); p < pend; ++p) { s << ", " << *p; } @@ -66,17 +66,13 @@ std::ostream& operator<< (std::ostream& s, const object o) case type::MAP: s << "{"; - if(o.via.container.size != 0) { - object* p(o.via.container.ptr); - object* const pend(o.via.container.ptr + o.via.container.size*2); - s << *p; ++p; - s << "=>"; - s << *p; ++p; - while(p < pend) { - s << ", "; - s << *p; ++p; - s << "=>"; - s << *p; ++p; + if(o.via.map.size != 0) { + object_kv* p(o.via.map.ptr); + s << p->key << "=>" << p->val; + ++p; + for(object_kv* const pend(o.via.map.ptr + o.via.map.size); + p < pend; ++p) { + s << ", " << p->key << "=>" << p->val; } } s << "}"; @@ -109,14 +105,14 @@ bool operator==(const object x, const object y) return x.via.i64 == y.via.i64; case type::RAW: - return x.via.ref.size == y.via.ref.size && - memcmp(x.via.ref.ptr, y.via.ref.ptr, x.via.ref.size) == 0; + return x.via.raw.size == y.via.raw.size && + memcmp(x.via.raw.ptr, y.via.raw.ptr, x.via.raw.size) == 0; case type::ARRAY: - if(x.via.container.size != y.via.container.size) { return false; } - for(object* px(x.via.container.ptr), - * const pxend(x.via.container.ptr + x.via.container.size), - * py(y.via.container.ptr); + if(x.via.array.size != y.via.array.size) { return false; } + for(object* px(x.via.array.ptr), + * const pxend(x.via.array.ptr + x.via.array.size), + * py(y.via.array.ptr); px < pxend; ++px, ++py) { if(*px != *py) { return false; } } @@ -124,12 +120,12 @@ bool operator==(const object x, const object y) // FIXME loop optimiziation case type::MAP: - if(x.via.container.size != y.via.container.size) { return false; } - for(object* px(x.via.container.ptr), - * const pxend(x.via.container.ptr + x.via.container.size*2), - * py(y.via.container.ptr); + if(x.via.map.size != y.via.map.size) { return false; } + for(object_kv* px(x.via.map.ptr), + * const pxend(x.via.map.ptr + x.via.map.size), + * py(y.via.map.ptr); px < pxend; ++px, ++py) { - if(*px != *py) { return false; } + if(px->key != py->key || px->val != py->val) { return false; } } return true; diff --git a/cpp/object.hpp b/cpp/object.hpp index c6f1e4a..08e715d 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -45,20 +45,33 @@ namespace type { } +struct object; +struct object_kv; + +struct object_array { + uint32_t size; + object* ptr; +}; + +struct object_map { + uint32_t size; + object_kv* ptr; +}; + +struct object_raw { + uint32_t size; + const char* ptr; +}; + struct object { union union_type { bool boolean; uint64_t u64; int64_t i64; double dec; - struct { - object* ptr; - uint32_t size; - } container; - struct { - const char* ptr; - uint32_t size; - } ref; + object_array array; + object_map map; + object_raw raw; }; type::object_type type; @@ -79,6 +92,11 @@ public: implicit_type convert(); }; +struct object_kv { + object key; + object val; +}; + bool operator==(const object x, const object y); bool operator!=(const object x, const object y); @@ -258,14 +276,14 @@ packer& operator<< (packer& o, const object& v) return o; case type::RAW: - o.pack_raw(v.via.ref.size); - o.pack_raw_body(v.via.ref.ptr, v.via.ref.size); + o.pack_raw(v.via.raw.size); + o.pack_raw_body(v.via.raw.ptr, v.via.raw.size); return o; case type::ARRAY: - o.pack_array(v.via.container.size); - for(object* p(v.via.container.ptr), - * const pend(v.via.container.ptr + v.via.container.size); + o.pack_array(v.via.array.size); + for(object* p(v.via.array.ptr), + * const pend(v.via.array.ptr + v.via.array.size); p < pend; ++p) { *p >> o; } @@ -273,12 +291,12 @@ packer& operator<< (packer& o, const object& v) // FIXME loop optimiziation case type::MAP: - o.pack_map(v.via.container.size); - for(object* p(v.via.container.ptr), - * const pend(v.via.container.ptr + v.via.container.size*2); - p < pend; ) { - *p >> o; ++p; - *p >> o; ++p; + o.pack_map(v.via.map.size); + for(object_kv* p(v.via.map.ptr), + * const pend(v.via.map.ptr + v.via.map.size); + p < pend; ++p) { + p->key >> o; + p->val >> o; } return o; // FIXME loop optimiziation diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp index f9f8038..b2a81ef 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/array.hpp @@ -28,9 +28,9 @@ template inline std::vector operator>> (object o, std::vector& v) { if(o.type != type::ARRAY) { throw type_error(); } - v.resize(o.via.container.size); - object* p = o.via.container.ptr; - object* const pend = o.via.container.ptr + o.via.container.size; + v.resize(o.via.array.size); + object* p = o.via.array.ptr; + object* const pend = o.via.array.ptr + o.via.array.size; T* it = &v.front(); for(; p < pend; ++p, ++it) { p->convert(it); diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index 619c36e..4585d66 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -46,13 +46,13 @@ template inline type::assoc_vector& operator>> (object o, type::assoc_vector& v) { if(o.type != type::MAP) { throw type_error(); } - v.resize(o.via.container.size); - object* p = o.via.container.ptr; - object* const pend = o.via.container.ptr + o.via.container.size; + v.resize(o.via.map.size); + object_kv* p = o.via.map.ptr; + object_kv* const pend = o.via.map.ptr + o.via.map.size; std::pair* it(&v.front()); - for(; p < pend; ++it) { - p->convert(&it->first); ++p; - p->convert(&it->second); ++p; + for(; p < pend; ++p, ++it) { + p->key.convert(&it->first); + p->val.convert(&it->second); } std::sort(v.begin(), v.end(), type::detail::pair_first_less()); return v; @@ -75,18 +75,18 @@ template inline std::map operator>> (object o, std::map& v) { if(o.type != type::MAP) { throw type_error(); } - object* p(o.via.container.ptr); - object* const pend(o.via.container.ptr + o.via.container.size*2); - while(p < pend) { + object_kv* p(o.via.map.ptr); + object_kv* const pend(o.via.map.ptr + o.via.map.size); + for(; p != pend; ++p) { K key; - p->convert(&key); ++p; + p->key.convert(&key); typename std::map::iterator it(v.find(key)); if(it != v.end()) { V val; - p->convert(&val); ++p; + p->val.convert(&val); it->insert( std::pair(key, val) ); } else { - p->convert(&it->second); ++p; + p->val.convert(&it->second); } } return v; @@ -109,12 +109,12 @@ template inline std::multimap operator>> (object o, std::multimap& v) { if(o.type != type::MAP) { throw type_error(); } - object* p = o.via.container.ptr; - object* const pend = o.via.container.ptr + o.via.container.size*2; - while(p < pend) { + object_kv* p(o.via.map.ptr); + object_kv* const pend(o.via.map.ptr + o.via.map.size); + for(; p != pend; ++p) { std::pair value; - p->convert(&value.first); ++p; - p->convert(&value.second); ++p; + p->key.convert(&value.first); + p->val.convert(&value.second); v.insert(value); } return v; @@ -123,7 +123,7 @@ inline std::multimap operator>> (object o, std::multimap& v) template inline packer& operator<< (packer& o, const std::multimap& v) { - o.pack_multimap(v.size()); + o.pack_map(v.size()); for(typename std::multimap::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { o.pack(it->first); diff --git a/cpp/type/raw.hpp b/cpp/type/raw.hpp index e4f9dd3..b102ee8 100644 --- a/cpp/type/raw.hpp +++ b/cpp/type/raw.hpp @@ -27,11 +27,11 @@ namespace msgpack { namespace type { struct raw_ref { - raw_ref() : ptr(NULL), size(0) {} - raw_ref(const char* p, uint32_t s) : ptr(p), size(s) {} + raw_ref() : size(0), ptr(NULL) {} + raw_ref(const char* p, uint32_t s) : size(s), ptr(p) {} - const char* ptr; uint32_t size; + const char* ptr; std::string str() { return std::string(ptr, size); } @@ -64,8 +64,8 @@ struct raw_ref { inline type::raw_ref& operator>> (object o, type::raw_ref& v) { if(o.type != type::RAW) { throw type_error(); } - v.ptr = o.via.ref.ptr; - v.size = o.via.ref.size; + v.ptr = o.via.raw.ptr; + v.size = o.via.raw.size; return v; } diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index a66c57e..a20f5d9 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -136,9 +136,9 @@ type::tuple, A<%=j%><%}%>>& operator>> ( object o, type::tuple, A<%=j%><%}%>>& v) { if(o.type != type::ARRAY) { throw type_error(); } - if(o.via.container.size < <%=i+1%>) { throw type_error(); } + if(o.via.array.size < <%=i+1%>) { throw type_error(); } <%0.upto(i) {|j|%> - o.via.container.ptr[<%=j%>].convert>(&v.template get<<%=j%>>());<%}%> + o.via.array.ptr[<%=j%>].convert>(&v.template get<<%=j%>>());<%}%> return v; } <%}%> diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 62088d6..0b9a476 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -104,36 +104,36 @@ static inline object msgpack_unpack_array(unpack_user* u, unsigned int n) { object o; o.type = type::ARRAY; - o.via.container.size = 0; - o.via.container.ptr = (object*)u->z->malloc(n*sizeof(object)); + o.via.array.size = 0; + o.via.array.ptr = (object*)u->z->malloc(n*sizeof(object)); return o; } static inline void msgpack_unpack_array_item(unpack_user* u, object* c, object o) -{ c->via.container.ptr[ c->via.container.size++ ] = o; } +{ c->via.array.ptr[c->via.array.size++] = o; } static inline object msgpack_unpack_map(unpack_user* u, unsigned int n) { object o; o.type = type::MAP; - o.via.container.size = 0; - o.via.container.ptr = (object*)u->z->malloc(n*2*sizeof(object)); + o.via.map.size = 0; + o.via.map.ptr = (object_kv*)u->z->malloc(n*sizeof(object_kv)); return o; } static inline void msgpack_unpack_map_item(unpack_user* u, object* c, object k, object v) { - c->via.container.ptr[ c->via.container.size ] = k; - c->via.container.ptr[ c->via.container.size+1 ] = v; - ++c->via.container.size; + c->via.map.ptr[c->via.map.size].key = k; + c->via.map.ptr[c->via.map.size].val = v; + ++c->via.map.size; } static inline object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) { object o; o.type = type::RAW; - o.via.ref.ptr = p; - o.via.ref.size = l; + o.via.raw.ptr = p; + o.via.raw.size = l; u->referenced = true; return o; } @@ -332,7 +332,7 @@ object unpacker::data() void unpacker::reset() { - if(!m_zone->empty()) { delete release_zone(); } + //if(!m_zone->empty()) { delete release_zone(); } as_ctx(m_ctx)->reset(); } From cbf7afc1ccbbabb8230a1a076eb2251aff7919e0 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Sun, 22 Feb 2009 15:14:21 +0900 Subject: [PATCH 0057/1172] c and c++: rewritten and integrated --- COPYING | 2 +- Makefile.am | 5 + README | 2 +- bootstrap | 8 +- c/AUTHORS | 1 - c/COPYING | 14 -- c/ChangeLog | 0 c/NEWS | 0 c/README | 21 --- c/bootstrap | 3 - c/configure.in | 16 -- c/object.c | 199 ++++++-------------- c/object.h | 23 +-- c/pack.h | 87 +++++---- c/unpack.c | 384 +++++++++++++++++++++++++++++++------- c/unpack.h | 103 ++++++---- c/zone.c | 253 +++++++++++++++++++------ c/zone.h | 49 ++++- configure.in | 38 +++- cpp/AUTHORS | 1 - cpp/COPYING | 14 -- cpp/ChangeLog | 0 cpp/Makefile.am | 7 +- cpp/NEWS | 0 cpp/README | 21 --- cpp/bootstrap | 3 - cpp/configure.in | 23 --- cpp/type/array.hpp | 2 +- cpp/type/boolean.hpp | 2 +- cpp/type/float.hpp | 2 +- cpp/type/integer.hpp | 2 +- cpp/type/map.hpp | 2 +- cpp/type/nil.hpp | 2 +- cpp/type/raw.hpp | 2 +- cpp/type/tuple.hpp.erb | 2 +- cpp/unpack.cpp | 369 ------------------------------------ cpp/unpack.hpp | 171 ++++++++++++----- cpp/zone.cpp | 100 ---------- cpp/zone.hpp.erb | 59 +++--- example/stream.cc | 1 + msgpack/unpack_template.h | 26 +-- ruby/gem/README | 4 +- ruby/gengem.sh | 2 + ruby/unpack.c | 134 ++++++------- 44 files changed, 1035 insertions(+), 1124 deletions(-) delete mode 100644 c/AUTHORS delete mode 100644 c/COPYING delete mode 100644 c/ChangeLog delete mode 100644 c/NEWS delete mode 100644 c/README delete mode 100755 c/bootstrap delete mode 100644 c/configure.in delete mode 100644 cpp/AUTHORS delete mode 100644 cpp/COPYING delete mode 100644 cpp/ChangeLog delete mode 100644 cpp/NEWS delete mode 100644 cpp/README delete mode 100755 cpp/bootstrap delete mode 100644 cpp/configure.in delete mode 100644 cpp/unpack.cpp delete mode 100644 cpp/zone.cpp diff --git a/COPYING b/COPYING index 5f100cd..6f5f220 100644 --- a/COPYING +++ b/COPYING @@ -1,4 +1,4 @@ -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Makefile.am b/Makefile.am index c23320a..3144972 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,9 @@ +if ENABLE_CXX +export ERB SUBDIRS = c cpp +else +SUBDIRS = c +endif nobase_include_HEADERS = \ msgpack/pack_define.h \ diff --git a/README b/README index 87d9279..fd5fa8d 100644 --- a/README +++ b/README @@ -54,7 +54,7 @@ Binary-based efficient data interchange format. -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/bootstrap b/bootstrap index 954fa89..eb16c35 100755 --- a/bootstrap +++ b/bootstrap @@ -31,10 +31,10 @@ if test x"$1" = x"--help"; then fi -if [ -z "$NO_NEST" ];then - cd c && ./bootstrap $@; cd .. - cd cpp && ./bootstrap $@; cd .. -fi +#if [ -z "$NO_NEST" ];then +# cd c && ./bootstrap $@; cd .. +# cd cpp && ./bootstrap $@; cd .. +#fi ACLOCAL="aclocal" diff --git a/c/AUTHORS b/c/AUTHORS deleted file mode 100644 index ababacb..0000000 --- a/c/AUTHORS +++ /dev/null @@ -1 +0,0 @@ -FURUHASHI Sadayuki diff --git a/c/COPYING b/c/COPYING deleted file mode 100644 index 6f5f220..0000000 --- a/c/COPYING +++ /dev/null @@ -1,14 +0,0 @@ -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/c/ChangeLog b/c/ChangeLog deleted file mode 100644 index e69de29..0000000 diff --git a/c/NEWS b/c/NEWS deleted file mode 100644 index e69de29..0000000 diff --git a/c/README b/c/README deleted file mode 100644 index 76dc221..0000000 --- a/c/README +++ /dev/null @@ -1,21 +0,0 @@ -MessagePack for C ------------------ -MessagePack is a binary-based efficient data interchange format. - - - -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - diff --git a/c/bootstrap b/c/bootstrap deleted file mode 100755 index 7e61b82..0000000 --- a/c/bootstrap +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -NO_NEST=1 -. ../bootstrap diff --git a/c/configure.in b/c/configure.in deleted file mode 100644 index 47eb8aa..0000000 --- a/c/configure.in +++ /dev/null @@ -1,16 +0,0 @@ -AC_INIT(unpack.c) -AM_INIT_AUTOMAKE(msgpackc, 0.1.0) -AC_CONFIG_HEADER(config.h) - -AC_SUBST(CFLAGS) -if test "" = "$CFLAGS"; then - CFLAGS="-g -O4" -fi - -AC_PROG_CC -AC_PROG_LIBTOOL - -CFLAGS="-O4 -Wall $CFLAGS -I.." - -AC_OUTPUT([Makefile]) - diff --git a/c/object.c b/c/object.c index 6376438..871153d 100644 --- a/c/object.c +++ b/c/object.c @@ -15,153 +15,78 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "msgpack/unpack.h" -#include "msgpack/unpack_define.h" #include "msgpack/object.h" +#include +#include -typedef struct { - msgpack_zone* z; - bool referenced; - bool failed; -} unpack_user; - -#define msgpack_unpack_struct(name) \ - struct msgpack_unpacker ## name - -#define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker ## name - -#define msgpack_unpack_callback(name) \ - msgpack_unpack ## name - -#define msgpack_unpack_object msgpack_object - -#define msgpack_unpack_user unpack_user - - -struct msgpack_unpacker_context; - -static void msgpack_unpacker_init(struct msgpack_unpacker_context* ctx); - -static msgpack_object msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); - -static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, - const char* data, size_t len, size_t* off); - -static inline msgpack_object msgpack_unpack_init(unpack_user* u) -{ msgpack_object o; return o; } - -static inline msgpack_object msgpack_unpack_uint8(unpack_user* u, uint8_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_uint16(unpack_user* u, uint16_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_uint32(unpack_user* u, uint32_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_uint64(unpack_user* u, uint64_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_int8(unpack_user* u, int8_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_int16(unpack_user* u, int16_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_int32(unpack_user* u, int32_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_int64(unpack_user* u, int64_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_float(unpack_user* u, float d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_DOUBLE; o.via.dec = d; return o; } - -static inline msgpack_object msgpack_unpack_double(unpack_user* u, double d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_DOUBLE; o.via.dec = d; return o; } - -static inline msgpack_object msgpack_unpack_nil(unpack_user* u) -{ msgpack_object o; o.type = MSGPACK_OBJECT_NIL; return o; } - -static inline msgpack_object msgpack_unpack_true(unpack_user* u) -{ msgpack_object o; o.type = MSGPACK_OBJECT_BOOLEAN; o.via.boolean = true; return o; } - -static inline msgpack_object msgpack_unpack_false(unpack_user* u) -{ msgpack_object o; o.type = MSGPACK_OBJECT_BOOLEAN; o.via.boolean = false; return o; } - -static inline msgpack_object msgpack_unpack_array(unpack_user* u, unsigned int n) +void msgpack_object_print(FILE* out, msgpack_object o) { - msgpack_object o; - o.type = MSGPACK_OBJECT_ARRAY; - o.via.array.size = 0; - o.via.array.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); - if(o.via.array.ptr == NULL) { u->failed = true; } - return o; -} + switch(o.type) { + case MSGPACK_OBJECT_NIL: + fprintf(out, "nil"); + break; -static inline void msgpack_unpack_array_item(unpack_user* u, msgpack_object* c, msgpack_object o) -{ - if(u->failed) { return; } - c->via.array.ptr[ c->via.array.size++ ] = o; -} + case MSGPACK_OBJECT_BOOLEAN: + fprintf(out, (o.via.boolean ? "true" : "false")); + break; -static inline msgpack_object msgpack_unpack_map(unpack_user* u, unsigned int n) -{ - msgpack_object o; - o.type = MSGPACK_OBJECT_MAP; - o.via.map.size = 0; - o.via.map.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object_kv)); - if(o.via.map.ptr == NULL) { u->failed = true; } - return o; -} + case MSGPACK_OBJECT_POSITIVE_INTEGER: + fprintf(out, "%"PRIu64, o.via.u64); + break; -static inline void msgpack_unpack_map_item(unpack_user* u, msgpack_object* c, msgpack_object k, msgpack_object v) -{ - if(u->failed) { return; } - c->via.map.ptr[c->via.map.size].key = k; - c->via.map.ptr[c->via.map.size].val = v; - ++c->via.map.size; -} + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + fprintf(out, "%"PRIi64, o.via.i64); + break; -static inline msgpack_object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) -{ - msgpack_object o; - o.type = MSGPACK_OBJECT_RAW; - o.via.raw.ptr = p; - o.via.raw.size = l; - u->referenced = true; - return o; -} + case MSGPACK_OBJECT_DOUBLE: + fprintf(out, "%f", o.via.dec); + break; -#include "msgpack/unpack_template.h" + case MSGPACK_OBJECT_RAW: + fprintf(out, "\""); + fwrite(o.via.raw.ptr, o.via.raw.size, 1, out); + fprintf(out, "\""); + break; -msgpack_object_unpack_return -msgpack_object_unpack(const char* data, size_t len, size_t* off, - msgpack_zone* z, msgpack_object* result) -{ - struct msgpack_unpacker_context ctx; - msgpack_unpacker_init(&ctx); - unpack_user u = {z, false, false}; - ctx.user = u; + case MSGPACK_OBJECT_ARRAY: + fprintf(out, "["); + if(o.via.array.size != 0) { + msgpack_object* p = o.via.array.ptr; + msgpack_object_print(out, *p); + ++p; + msgpack_object* const pend = o.via.array.ptr + o.via.array.size; + for(; p < pend; ++p) { + fprintf(out, ", "); + msgpack_object_print(out, *p); + } + } + fprintf(out, "]"); + break; + // FIXME loop optimiziation - size_t noff = (off ? *off : 0); - int ret = msgpack_unpacker_execute(&ctx, data, len, &noff); - if(ret < 0 || ctx.user.failed) { - return MSGPACK_OBJECT_PARSE_ERROR; - } else if(ret == 0) { - return MSGPACK_OBJECT_INSUFFICIENT_BYTES; + case MSGPACK_OBJECT_MAP: + fprintf(out, "{"); + if(o.via.map.size != 0) { + msgpack_object_kv* p = o.via.map.ptr; + msgpack_object_print(out, p->key); + fprintf(out, "=>"); + msgpack_object_print(out, p->val); + ++p; + msgpack_object_kv* const pend = o.via.map.ptr + o.via.map.size; + for(; p < pend; ++p) { + fprintf(out, ", "); + msgpack_object_print(out, p->key); + fprintf(out, "=>"); + msgpack_object_print(out, p->val); + } + } + fprintf(out, "}"); + break; + // FIXME loop optimiziation + + default: + // FIXME + fprintf(out, "#", o.type, o.via.u64); } - *result = msgpack_unpacker_data(&ctx); - if(off) { *off = noff; } - if(ret == 0) { - return MSGPACK_OBJECT_EXTRA_BYTES; - } - return MSGPACK_OBJECT_PARSE_SUCCESS; } - diff --git a/c/object.h b/c/object.h index b590ebe..7c603b3 100644 --- a/c/object.h +++ b/c/object.h @@ -22,6 +22,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -39,18 +40,18 @@ typedef enum { MSGPACK_OBJECT_MAP = 0x08, } msgpack_object_type; -struct _msgpack_object; -struct _msgpack_object_kv; +struct msgpack_object; +struct msgpack_object_kv; typedef struct { uint32_t size; - struct _msgpack_object* ptr; + struct msgpack_object* ptr; } msgpack_object_array; typedef struct { uint32_t size; - struct _msgpack_object_kv* ptr; + struct msgpack_object_kv* ptr; } msgpack_object_map; typedef struct { @@ -68,26 +69,18 @@ typedef union { msgpack_object_raw raw; } msgpack_object_union; -typedef struct _msgpack_object { +typedef struct msgpack_object { msgpack_object_type type; msgpack_object_union via; } msgpack_object; -typedef struct _msgpack_object_kv { +typedef struct msgpack_object_kv { msgpack_object key; msgpack_object val; } msgpack_object_kv; -typedef enum { - MSGPACK_OBJECT_PARSE_SUCCESS = 0, - MSGPACK_OBJECT_EXTRA_BYTES = 1, - MSGPACK_OBJECT_INSUFFICIENT_BYTES = -1, - MSGPACK_OBJECT_PARSE_ERROR = -2, -} msgpack_object_unpack_return; -msgpack_object_unpack_return -msgpack_object_unpack(const char* data, size_t len, size_t* off, - msgpack_zone* z, msgpack_object* result); +void msgpack_object_print(FILE* out, msgpack_object o); #ifdef __cplusplus diff --git a/c/pack.h b/c/pack.h index a510eff..46de722 100644 --- a/c/pack.h +++ b/c/pack.h @@ -28,50 +28,49 @@ extern "C" { #endif -typedef int (*msgpack_pack_write_t)(void* data, const char* buf, unsigned int len); +typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len); -typedef struct { +typedef struct msgpack_packer { void* data; - msgpack_pack_write_t callback; -} msgpack_pack_t; + msgpack_packer_write callback; +} msgpack_packer; -static void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback); +static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); -static msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback); +static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); +static void msgpack_packer_free(msgpack_packer* pk); -static void msgpack_pack_free(msgpack_pack_t* ctx); +static int msgpack_pack_short(msgpack_packer* pk, short d); +static int msgpack_pack_int(msgpack_packer* pk, int d); +static int msgpack_pack_long(msgpack_packer* pk, long d); +static int msgpack_pack_long_long(msgpack_packer* pk, long long d); +static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); +static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); +static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); +static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); -static int msgpack_pack_short(msgpack_pack_t* ctx, short d); -static int msgpack_pack_int(msgpack_pack_t* ctx, int d); -static int msgpack_pack_long(msgpack_pack_t* ctx, long d); -static int msgpack_pack_long_long(msgpack_pack_t* ctx, long long d); -static int msgpack_pack_unsigned_short(msgpack_pack_t* ctx, unsigned short d); -static int msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); -static int msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); -static int msgpack_pack_unsigned_long_long(msgpack_pack_t* ctx, unsigned long long d); +static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); +static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); +static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); +static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); +static int msgpack_pack_int8(msgpack_packer* pk, int8_t d); +static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); +static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); +static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); -static int msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); -static int msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); -static int msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); -static int msgpack_pack_uint64(msgpack_pack_t* ctx, uint64_t d); -static int msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); -static int msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); -static int msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); -static int msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); +static int msgpack_pack_float(msgpack_packer* pk, float d); +static int msgpack_pack_double(msgpack_packer* pk, double d); -static int msgpack_pack_float(msgpack_pack_t* ctx, float d); -static int msgpack_pack_double(msgpack_pack_t* ctx, double d); +static int msgpack_pack_nil(msgpack_packer* pk); +static int msgpack_pack_true(msgpack_packer* pk); +static int msgpack_pack_false(msgpack_packer* pk); -static int msgpack_pack_nil(msgpack_pack_t* ctx); -static int msgpack_pack_true(msgpack_pack_t* ctx); -static int msgpack_pack_false(msgpack_pack_t* ctx); +static int msgpack_pack_array(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); +static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); - -static int msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); -static int msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); +static int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); @@ -81,30 +80,30 @@ static int msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); #define msgpack_pack_inline_func_cint(name) \ inline int msgpack_pack ## name -#define msgpack_pack_user msgpack_pack_t* +#define msgpack_pack_user msgpack_packer* #define msgpack_pack_append_buffer(user, buf, len) \ return (*(user)->callback)((user)->data, (const char*)buf, len) #include "msgpack/pack_template.h" -inline void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback) +inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) { - ctx->data = data; - ctx->callback = callback; + pk->data = data; + pk->callback = callback; } -inline msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback) +inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback) { - msgpack_pack_t* ctx = (msgpack_pack_t*)calloc(1, sizeof(msgpack_pack_t)); - if(!ctx) { return NULL; } - msgpack_pack_init(ctx, data, callback); - return ctx; + msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer)); + if(!pk) { return NULL; } + msgpack_packer_init(pk, data, callback); + return pk; } -inline void msgpack_pack_free(msgpack_pack_t* ctx) +inline void msgpack_packer_free(msgpack_packer* pk) { - free(ctx); + free(pk); } diff --git a/c/unpack.c b/c/unpack.c index 03c67be..e06d97c 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -20,125 +20,371 @@ #include +typedef struct { + msgpack_zone* z; + bool* referenced; +} unpack_user; + + #define msgpack_unpack_struct(name) \ struct template ## name #define msgpack_unpack_func(ret, name) \ - ret template_func ## name + ret template ## name #define msgpack_unpack_callback(name) \ template_callback ## name -#define msgpack_unpack_object void* +#define msgpack_unpack_object msgpack_object -#define msgpack_unpack_user msgpack_unpack_t +#define msgpack_unpack_user unpack_user struct template_context; +typedef struct template_context template_context; -static void template_func_init(struct template_context* ctx); +static void template_init(template_context* ctx); -static void* template_func_data(struct template_context* ctx); +static msgpack_object template_data(template_context* ctx); -static int template_func_execute(struct template_context* ctx, +static int template_execute(template_context* ctx, const char* data, size_t len, size_t* off); -static inline void* template_callback_init(msgpack_unpack_t* x) -{ return NULL; } +static inline msgpack_object template_callback_root(unpack_user* u) +{ msgpack_object o; return o; } -static inline void* template_callback_uint8(msgpack_unpack_t* x, uint8_t d) -{ return x->callback.unpack_uint8(x->data, d); } +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_uint16(msgpack_unpack_t* x, uint16_t d) -{ return x->callback.unpack_uint16(x->data, d); } +static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_uint32(msgpack_unpack_t* x, uint32_t d) -{ return x->callback.unpack_uint32(x->data, d); } +static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_uint64(msgpack_unpack_t* x, uint64_t d) -{ return x->callback.unpack_uint64(x->data, d); } +static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_int8(msgpack_unpack_t* x, int8_t d) -{ return x->callback.unpack_int8(x->data, d); } +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_int16(msgpack_unpack_t* x, int16_t d) -{ return x->callback.unpack_int16(x->data, d); } +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_int32(msgpack_unpack_t* x, int32_t d) -{ return x->callback.unpack_int32(x->data, d); } +static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_int64(msgpack_unpack_t* x, int64_t d) -{ return x->callback.unpack_int64(x->data, d); } +static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_float(msgpack_unpack_t* x, float d) -{ return x->callback.unpack_float(x->data, d); } +static inline int template_callback_float(unpack_user* u, float d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_DOUBLE; o->via.dec = d; return 0; } -static inline void* template_callback_double(msgpack_unpack_t* x, double d) -{ return x->callback.unpack_double(x->data, d); } +static inline int template_callback_double(unpack_user* u, double d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_DOUBLE; o->via.dec = d; return 0; } -static inline void* template_callback_nil(msgpack_unpack_t* x) -{ return x->callback.unpack_nil(x->data); } +static inline int template_callback_nil(unpack_user* u, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_NIL; return 0; } -static inline void* template_callback_true(msgpack_unpack_t* x) -{ return x->callback.unpack_true(x->data); } +static inline int template_callback_true(unpack_user* u, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_BOOLEAN; o->via.boolean = true; return 0; } -static inline void* template_callback_false(msgpack_unpack_t* x) -{ return x->callback.unpack_false(x->data); } +static inline int template_callback_false(unpack_user* u, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_BOOLEAN; o->via.boolean = false; return 0; } -static inline void* template_callback_array(msgpack_unpack_t* x, unsigned int n) -{ return x->callback.unpack_array(x->data, n); } +static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_object* o) +{ + o->type = MSGPACK_OBJECT_ARRAY; + o->via.array.size = 0; + o->via.array.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); + if(o->via.array.ptr == NULL) { return -1; } + return 0; +} -static inline void template_callback_array_item(msgpack_unpack_t* x, void** c, void* o) -{ x->callback.unpack_array_item(x->data, *c, o); } +static inline int template_callback_array_item(unpack_user* u, msgpack_object* c, msgpack_object o) +{ c->via.array.ptr[c->via.array.size++] = o; return 0; } -static inline void* template_callback_map(msgpack_unpack_t* x, unsigned int n) -{ return x->callback.unpack_map(x->data, n); } +static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_object* o) +{ + o->type = MSGPACK_OBJECT_MAP; + o->via.map.size = 0; + o->via.map.ptr = (msgpack_object_kv*)msgpack_zone_malloc(u->z, n*sizeof(msgpack_object_kv)); + if(o->via.map.ptr == NULL) { return -1; } + return 0; +} -static inline void template_callback_map_item(msgpack_unpack_t* x, void** c, void* k, void* v) -{ x->callback.unpack_map_item(x->data, *c, k, v); } - -static inline void* template_callback_raw(msgpack_unpack_t* x, const char* b, const char* p, unsigned int l) -{ return x->callback.unpack_raw(x->data, b, p, l); } +static inline int template_callback_map_item(unpack_user* u, msgpack_object* c, msgpack_object k, msgpack_object v) +{ + c->via.map.ptr[c->via.map.size].key = k; + c->via.map.ptr[c->via.map.size].val = v; + ++c->via.map.size; + return 0; +} +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_object* o) +{ + o->type = MSGPACK_OBJECT_RAW; + o->via.raw.ptr = p; + o->via.raw.size = l; + *u->referenced = true; + return 0; +} #include "msgpack/unpack_template.h" -msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback) +#define CTX_CAST(m) ((template_context*)(m)) + + +static const size_t COUNTER_SIZE = sizeof(unsigned int); + +static inline void init_count(void* buf) { - struct template_context* ctx; - ctx = (struct template_context*)calloc(1, sizeof(struct template_context)); - if(ctx == NULL) { return NULL; } - template_func_init(ctx); - ((msgpack_unpack_t*)ctx)->data = data; - ((msgpack_unpack_t*)ctx)->callback = *callback; - return (msgpack_unpack_t*)ctx; + *(volatile unsigned int*)buf = 1; } -void msgpack_unpack_free(msgpack_unpack_t* ctx) +static inline void decl_count(void* buf) { - free((struct template_context*)ctx); + //if(--*(unsigned int*)buf == 0) { + if(__sync_sub_and_fetch((unsigned int*)buf, 1) == 0) { + free(buf); + } } -void* msgpack_unpack_data(msgpack_unpack_t* ctx) +static inline void incr_count(void* buf) { - return template_func_data((struct template_context*)ctx); + //++*(unsigned int*)buf; + __sync_add_and_fetch((unsigned int*)buf, 1); } -void msgpack_unpack_reset(msgpack_unpack_t* ctx) +static inline unsigned int get_count(void* buf) { - msgpack_unpack_t x = ((struct template_context*)ctx)->user; - template_func_init((struct template_context*)ctx); - ((struct template_context*)ctx)->user = x; -} - -int msgpack_unpack_execute(msgpack_unpack_t* ctx, - const char* data, size_t len, size_t* off) -{ - return template_func_execute( - (struct template_context*)ctx, - data, len, off); + return *(volatile unsigned int*)buf; } + +bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) +{ + if(initial_buffer_size < COUNTER_SIZE) { + initial_buffer_size = COUNTER_SIZE; + } + + char* buf = (char*)malloc(initial_buffer_size); + if(buf == NULL) { + return false; + } + + void* ctx = malloc(sizeof(template_context)); + if(ctx == NULL) { + free(buf); + return false; + } + + msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(z == NULL) { + free(ctx); + free(buf); + return false; + } + + mpac->buf = buf; + mpac->used = COUNTER_SIZE; + mpac->free = initial_buffer_size - mpac->used; + mpac->off = COUNTER_SIZE; + mpac->initial_buffer_size = initial_buffer_size; + mpac->z = z; + mpac->referenced = false; + mpac->ctx = ctx; + + init_count(mpac->buf); + + template_init(CTX_CAST(mpac->ctx)); + CTX_CAST(mpac->ctx)->user.z = mpac->z; + CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; + + return true; +} + +void msgpack_unpacker_destroy(msgpack_unpacker* mpac) +{ + msgpack_zone_free(mpac->z); + free(mpac->ctx); + decl_count(mpac->buf); +} + + +msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size) +{ + msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker)); + if(mpac == NULL) { + return NULL; + } + + if(!msgpack_unpacker_init(mpac, initial_buffer_size)) { + free(mpac); + return NULL; + } + + return mpac; +} + +void msgpack_unpacker_free(msgpack_unpacker* mpac) +{ + msgpack_unpacker_destroy(mpac); + free(mpac); +} + + +bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) +{ + if(mpac->used == mpac->off && get_count(mpac->buf) == 1 && !mpac->referenced) { + // rewind buffer + mpac->free += mpac->used - COUNTER_SIZE; + mpac->used = COUNTER_SIZE; + mpac->off = COUNTER_SIZE; + + if(mpac->free >= size) { + return true; + } + } + + if(mpac->off == COUNTER_SIZE) { + size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE + while(next_size < size + mpac->used) { + next_size *= 2; + } + + char* tmp = (char*)realloc(mpac->buf, next_size); + if(tmp == NULL) { + return false; + } + + mpac->buf = tmp; + mpac->free = next_size - mpac->used; + + } else { + size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE + size_t not_parsed = mpac->used - mpac->off; + while(next_size < size + not_parsed + COUNTER_SIZE) { + next_size *= 2; + } + + char* tmp = (char*)malloc(next_size); + if(tmp == NULL) { + return false; + } + + init_count(tmp); + + if(mpac->referenced) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + free(tmp); + return false; + } + mpac->referenced = false; + } else { + decl_count(mpac->buf); + } + + memcpy(tmp+COUNTER_SIZE, mpac->buf+mpac->off, not_parsed); + + mpac->buf = tmp; + mpac->used = not_parsed + COUNTER_SIZE; + mpac->free = next_size - mpac->used; + mpac->off = COUNTER_SIZE; + } + + return true; +} + +int msgpack_unpacker_execute(msgpack_unpacker* mpac) +{ + return template_execute(CTX_CAST(mpac->ctx), + mpac->buf, mpac->used, &mpac->off); +} + +msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) +{ + return template_data(CTX_CAST(mpac->ctx)); +} + +msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) +{ + if(!msgpack_unpacker_flush_zone(mpac)) { + return false; + } + + msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(z == NULL) { + return NULL; + } + + msgpack_zone* old = mpac->z; + mpac->z = z; + + return old; +} + +bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) +{ + if(mpac->referenced) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + return false; + } + mpac->referenced = false; + + incr_count(mpac->buf); + } + + return true; +} + +void msgpack_unpacker_reset(msgpack_unpacker* mpac) +{ + msgpack_zone* z = mpac->z; + template_init(CTX_CAST(mpac->ctx)); + CTX_CAST(mpac->ctx)->user.z = z; + CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; +} + + +msgpack_unpack_return +msgpack_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_object* result) +{ + template_context ctx; + template_init(&ctx); + + bool referenced = false; + ctx.user.z = z; + ctx.user.referenced = &referenced; + + size_t noff = 0; + if(off != NULL) { noff = *off; } + + int ret = template_execute(&ctx, data, len, &noff); + if(ret < 0) { + return MSGPACK_UNPACK_PARSE_ERROR; + } + + if(off != NULL) { *off = noff; } + + if(ret == 0) { + return MSGPACK_UNPACK_CONTINUE; + } + + *result = template_data(&ctx); + + if(noff < len) { + return MSGPACK_UNPACK_EXTRA_BYTES; + } + + return MSGPACK_UNPACK_SUCCESS; +} + diff --git a/c/unpack.h b/c/unpack.h index 4977f51..ab202a1 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -15,9 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MSGPACK_UNPACK_H__ -#define MSGPACK_UNPACK_H__ +#ifndef msgpack_unpacker_H__ +#define msgpack_unpacker_H__ +#include "msgpack/zone.h" +#include "msgpack/object.h" #include #include @@ -26,39 +28,76 @@ extern "C" { #endif -typedef struct { - void* (*unpack_uint8)(void* data, uint8_t d); - void* (*unpack_uint16)(void* data, uint16_t d); - void* (*unpack_uint32)(void* data, uint32_t d); - void* (*unpack_uint64)(void* data, uint64_t d); - void* (*unpack_int8)(void* data, int8_t d); - void* (*unpack_int16)(void* data, int16_t d); - void* (*unpack_int32)(void* data, int32_t d); - void* (*unpack_int64)(void* data, int64_t d); - void* (*unpack_float)(void* data, float d); - void* (*unpack_double)(void* data, double d); - void* (*unpack_nil)(void* data); - void* (*unpack_true)(void* data); - void* (*unpack_false)(void* data); - void* (*unpack_array)(void* data, unsigned int n); - void (*unpack_array_item)(void* data, void* c, void* o); - void* (*unpack_map)(void* data, unsigned int n); - void (*unpack_map_item)(void* data, void* c, void* k, void* v); - void* (*unpack_raw)(void* data, const char* b, const char* p, unsigned int l); -} msgpack_unpack_callback; +typedef struct msgpack_unpacker { + char* buf; + size_t used; + size_t free; + size_t off; + msgpack_zone* z; + bool referenced; + size_t initial_buffer_size; + void* ctx; +} msgpack_unpacker; -typedef struct { - void* data; - msgpack_unpack_callback callback; -} msgpack_unpack_t; -msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback); -void msgpack_unpack_free(msgpack_unpack_t* ctx); +bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size); +void msgpack_unpacker_destroy(msgpack_unpacker* mpac); -int msgpack_unpack_execute(msgpack_unpack_t* ctx, - const char* data, size_t len, size_t* off); -void* msgpack_unpack_data(msgpack_unpack_t* ctx); -void msgpack_unpack_reset(msgpack_unpack_t* ctx); +msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size); +void msgpack_unpacker_free(msgpack_unpacker* mpac); + +static inline bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size); +static inline char* msgpack_unpacker_buffer(msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac); +static inline void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size); + + +int msgpack_unpacker_execute(msgpack_unpacker* mpac); + +msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac); + +msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); + +void msgpack_unpacker_reset(msgpack_unpacker* mpac); + + +typedef enum { + MSGPACK_UNPACK_SUCCESS = 2, + MSGPACK_UNPACK_EXTRA_BYTES = 1, + MSGPACK_UNPACK_CONTINUE = 0, + MSGPACK_UNPACK_PARSE_ERROR = -1, +} msgpack_unpack_return; + +msgpack_unpack_return +msgpack_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_object* result); + + +bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); + +bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); + +bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) +{ + if(mpac->free >= size) { return true; } + return msgpack_unpacker_expand_buffer(mpac, size); +} + +char* msgpack_unpacker_buffer(msgpack_unpacker* mpac) +{ + return mpac->buf + mpac->used; +} + +size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac) +{ + return mpac->free; +} + +void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size) +{ + mpac->used += size; + mpac->free -= size; +} #ifdef __cplusplus diff --git a/c/zone.c b/c/zone.c index ccd702d..e891c82 100644 --- a/c/zone.c +++ b/c/zone.c @@ -19,85 +19,214 @@ #include #include -typedef struct { - size_t free; - void* ptr; - void* alloc; -} msgpack_zone_chunk; -struct _msgpack_zone { - msgpack_zone_chunk* array; - size_t ntail; - size_t usable; -}; - -msgpack_zone* msgpack_zone_new() +static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_size) { - return calloc(1, sizeof(msgpack_zone)); -} + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + const size_t nfirst = (sizeof(msgpack_zone_chunk) < 72/2) ? + 72 / sizeof(msgpack_zone_chunk) : 8; -void msgpack_zone_free(msgpack_zone* z) -{ - if(z->array) { - size_t i; - for(i=0; i <= z->ntail; ++i) { - free(z->array[i].alloc); - } + msgpack_zone_chunk* array = (msgpack_zone_chunk*)malloc( + sizeof(msgpack_zone_chunk) * nfirst); + if(!array) { + return false; } - free(z); + + const size_t sz = chunk_size; + + char* ptr = (char*)malloc(sz); + if(!ptr) { + free(array); + return NULL; + } + + ca->tail = array; + ca->end = array + nfirst; + ca->array = array; + + array[0].free = sz; + array[0].ptr = ptr; + array[0].alloc = ptr; + + return true; } - -void* msgpack_zone_malloc(msgpack_zone* z, size_t size) +static inline void destroy_chunk_array(msgpack_zone_chunk_array* ca) { - if(!z->array) { - const size_t n = (sizeof(msgpack_zone_chunk) < 72/2) ? - 72 / sizeof(msgpack_zone_chunk) : 8; - msgpack_zone_chunk* array = - (msgpack_zone_chunk*)malloc(sizeof(msgpack_zone_chunk) * n); - if(!array) { return NULL; } + msgpack_zone_chunk* chunk = ca->array; + for(; chunk != ca->tail+1; ++chunk) { + free(chunk->alloc); + } + free(ca->array); +} - size_t sz = 2048; /* FIXME chunk_size */ - while(sz < size) { sz *= 2; } - char* p = (char*)malloc(sz); - if(!p) { - free(array); +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +{ + msgpack_zone_chunk_array* const ca = &zone->chunk_array; + + msgpack_zone_chunk* chunk = ca->tail; + + if(chunk->free > size) { + // chunkã«ç©ºã容é‡ãŒã‚ã‚‹ + // 空ã容é‡ã‚’消費ã—ã¦è¿”ã™ + + char* ptr = chunk->ptr; + + chunk->ptr += size; + chunk->free -= size; + + return ptr; + } + + chunk = ++ca->tail; + + if(chunk == ca->end) { + // ca->arrayã«ç©ºããŒãªã„ + // ca->arrayã‚’æ‹¡å¼µã™ã‚‹ + + const size_t nused = ca->end - ca->array; + const size_t nnext = (ca->end - ca->array) * 2; + + chunk = (msgpack_zone_chunk*)realloc(ca->array, + sizeof(msgpack_zone_chunk) * nnext); + if(!chunk) { return NULL; } - z->array = array; - z->usable = n - 1; - array[0].free = sz - size; - array[0].ptr = p + size; - array[0].alloc = p; - return p; + ca->array = chunk; + ca->end = chunk + nnext; + chunk = ca->tail = chunk + nused; } - if(z->array[z->ntail].free > size) { - char* p = (char*)z->array[z->ntail].ptr; - z->array[z->ntail].ptr = p + size; - z->array[z->ntail].free -= size; - return p; + size_t sz = zone->chunk_size; + + while(sz < size) { + sz *= 2; } - if(z->usable <= z->ntail) { - const size_t n = (z->usable + 1) * 2; - msgpack_zone_chunk* tmp = - (msgpack_zone_chunk*)realloc(z->array, sizeof(msgpack_zone_chunk) * n); - if(!tmp) { return NULL; } - z->array = tmp; - z->usable = n - 1; + char* ptr = (char*)malloc(sz); + if(!ptr) { + return NULL; } - size_t sz = 2048; /* FIXME chunk_size */ - while(sz < size) { sz *= 2; } - char* p = (char*)malloc(sz); - if(!p) { return NULL; } + chunk->free = sz - size; + chunk->ptr = ptr + size; + chunk->alloc = ptr; - ++z->ntail; - z->array[z->ntail].free = sz - size; - z->array[z->ntail].ptr = p + size; - z->array[z->ntail].alloc = p; - return p; + return ptr; +} + + +static inline void init_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + fa->tail = NULL; + fa->end = NULL; + fa->array = NULL; +} + +static inline void destroy_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + // 逆順ã«å‘¼ã³å‡ºã— + msgpack_zone_finalizer* fin = fa->tail; + for(; fin != fa->array; --fin) { + (*(fin-1)->func)((fin-1)->data); + } + free(fa->array); +} + +bool msgpack_zone_push_finalizer(msgpack_zone* zone, + void (*func)(void* data), void* data) +{ + msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; + + msgpack_zone_finalizer* fin = fa->tail; + + if(fin == fa->end) { + // fa->arrayã«ç©ºããŒãªã„ + // fa->arrayã‚’æ‹¡å¼µã™ã‚‹ + + size_t nnext; + const size_t nused = fa->end - fa->array; + + if(nused == 0) { + // åˆå›žã®å‘¼ã³å‡ºã—：fa->tail == fa->end == fa->array == NULL + + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + nnext = (sizeof(msgpack_zone_finalizer) < 72/2) ? + 72 / sizeof(msgpack_zone_finalizer) : 8; + + } else { + nnext = (fa->end - fa->array) * 2; + } + + fin = (msgpack_zone_finalizer*)realloc(fa->array, + sizeof(msgpack_zone_finalizer) * nnext); + if(!fin) { + return false; + } + + fa->array = fin; + fa->end = fin + nnext; + fin = fa->tail = fin + nused; + } + + fin->func = func; + fin->data = data; + + ++fa->tail; + + return true; +} + + +bool msgpack_zone_is_empty(msgpack_zone* zone) +{ + msgpack_zone_chunk_array* const ca = &zone->chunk_array; + msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; + return ca->array[0].ptr == ca->array[0].alloc && + ca->tail == ca->array && + fa->tail == fa->array; +} + + +bool msgpack_zone_init(msgpack_zone* zone, size_t chunk_size) +{ + zone->chunk_size = chunk_size; + + if(!init_chunk_array(&zone->chunk_array, chunk_size)) { + return false; + } + + init_finalizer_array(&zone->finalizer_array); + + return true; +} + +void msgpack_zone_destroy(msgpack_zone* zone) +{ + destroy_finalizer_array(&zone->finalizer_array); + destroy_chunk_array(&zone->chunk_array); +} + + +msgpack_zone* msgpack_zone_new(size_t chunk_size) +{ + msgpack_zone* zone = (msgpack_zone*)malloc(sizeof(msgpack_zone)); + if(zone == NULL) { + return NULL; + } + + if(!msgpack_zone_init(zone, chunk_size)) { + free(zone); + return NULL; + } + + return zone; +} + +void msgpack_zone_free(msgpack_zone* zone) +{ + msgpack_zone_destroy(zone); + free(zone); } diff --git a/c/zone.h b/c/zone.h index ff6ca62..3dc9f52 100644 --- a/c/zone.h +++ b/c/zone.h @@ -19,19 +19,58 @@ #define MSGPACK_ZONE_H__ #include +#include #ifdef __cplusplus extern "C" { #endif -struct _msgpack_zone; -typedef struct _msgpack_zone msgpack_zone; +typedef struct msgpack_zone_chunk { + size_t free; + char* ptr; + void* alloc; +} msgpack_zone_chunk; -msgpack_zone* msgpack_zone_new(); -void msgpack_zone_free(msgpack_zone* z); +typedef struct msgpack_zone_finalizer { + void (*func)(void* data); + void* data; +} msgpack_zone_finalizer; -void* msgpack_zone_malloc(msgpack_zone* z, size_t size); +typedef struct msgpack_zone_chunk_array { + msgpack_zone_chunk* tail; + msgpack_zone_chunk* end; + msgpack_zone_chunk* array; +} msgpack_zone_chunk_array; + +typedef struct msgpack_zone_finalizer_array { + msgpack_zone_finalizer* tail; + msgpack_zone_finalizer* end; + msgpack_zone_finalizer* array; +} msgpack_zone_finalizer_array; + +typedef struct msgpack_zone { + msgpack_zone_chunk_array chunk_array; + msgpack_zone_finalizer_array finalizer_array; + size_t chunk_size; +} msgpack_zone; + +#ifndef MSGPACK_ZONE_CHUNK_SIZE +#define MSGPACK_ZONE_CHUNK_SIZE 2048 +#endif + +bool msgpack_zone_init(msgpack_zone* zone, size_t chunk_size); +void msgpack_zone_destroy(msgpack_zone* zone); + +msgpack_zone* msgpack_zone_new(size_t chunk_size); +void msgpack_zone_free(msgpack_zone* zone); + +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); + +bool msgpack_zone_push_finalizer(msgpack_zone* zone, + void (*func)(void* data), void* data); + +bool msgpack_zone_is_empty(msgpack_zone* zone); #ifdef __cplusplus diff --git a/configure.in b/configure.in index a464fb9..9fa6ac5 100644 --- a/configure.in +++ b/configure.in @@ -2,8 +2,42 @@ AC_INIT(msgpack/unpack_template.h) AM_INIT_AUTOMAKE(msgpack, 0.3.0) AC_CONFIG_HEADER(config.h) +AC_SUBST(CFLAGS) +if test "" = "$CFLAGS"; then + CFLAGS="-g -O4" +fi + +AC_PROG_CC + +CFLAGS="-O4 -Wall $CFLAGS -I.." + + +AC_MSG_CHECKING([if c++ api is enabled]) +AC_ARG_ENABLE(cxx, + AS_HELP_STRING([--disable-cxx], + [don't build c++ api.]) ) +AC_MSG_RESULT($enable_cxx) +if test "$enable_cxx" != "no"; then + AC_SUBST(CXXFLAGS) + if test "" = "$CXXFLAGS"; then + CXXFLAGS="-g -O4" + fi + + AC_CHECK_PROG(ERB, erb, erb) + if test "x$ERB" = x; then + AC_MSG_ERROR([cannot find erb. Ruby is needed to build.]) + fi +fi + +# FIXME +AC_PROG_CXX + +CXXFLAGS="-O4 -Wall $CXXFLAGS -I.." + + +AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no") + AC_PROG_LIBTOOL -AC_CONFIG_SUBDIRS([c cpp]) -AC_OUTPUT([Makefile]) +AC_OUTPUT([Makefile c/Makefile cpp/Makefile]) diff --git a/cpp/AUTHORS b/cpp/AUTHORS deleted file mode 100644 index ababacb..0000000 --- a/cpp/AUTHORS +++ /dev/null @@ -1 +0,0 @@ -FURUHASHI Sadayuki diff --git a/cpp/COPYING b/cpp/COPYING deleted file mode 100644 index 6f5f220..0000000 --- a/cpp/COPYING +++ /dev/null @@ -1,14 +0,0 @@ -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/cpp/ChangeLog b/cpp/ChangeLog deleted file mode 100644 index e69de29..0000000 diff --git a/cpp/Makefile.am b/cpp/Makefile.am index c7ddf4b..406c57b 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -1,9 +1,7 @@ lib_LTLIBRARIES = libmsgpack.la libmsgpack_la_SOURCES = \ - object.cpp \ - unpack.cpp \ - zone.cpp + object.cpp nobase_include_HEADERS = \ msgpack.hpp \ @@ -28,7 +26,6 @@ noinst_HEADERS = \ # FIXME object.lo: msgpack/type/tuple.hpp unpack.lo: msgpack/type/tuple.hpp msgpack/zone.hpp -zone.lo: msgpack/type/tuple.hpp msgpack/zone.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb $(ERB) $< > $@.tmp @@ -42,6 +39,8 @@ MOSTLYCLEANFILES = \ msgpack/type/tuple.hpp \ msgpack/zone.hpp +libmsgpack_la_LIBADD = -L../c -lmsgpackc + # -version-info CURRENT:REVISION:AGE libmsgpack_la_LDFLAGS = -version-info 1:0:0 diff --git a/cpp/NEWS b/cpp/NEWS deleted file mode 100644 index e69de29..0000000 diff --git a/cpp/README b/cpp/README deleted file mode 100644 index 96f18b1..0000000 --- a/cpp/README +++ /dev/null @@ -1,21 +0,0 @@ -MessagePack for C++ -------------------- -MessagePack is a binary-based efficient data interchange format. - - - -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - diff --git a/cpp/bootstrap b/cpp/bootstrap deleted file mode 100755 index 7e61b82..0000000 --- a/cpp/bootstrap +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -NO_NEST=1 -. ../bootstrap diff --git a/cpp/configure.in b/cpp/configure.in deleted file mode 100644 index 2c3e5d0..0000000 --- a/cpp/configure.in +++ /dev/null @@ -1,23 +0,0 @@ -AC_INIT(object.cpp) -AM_INIT_AUTOMAKE(msgpack, 0.1.0) -AC_CONFIG_HEADER(config.h) - -AC_SUBST(CXXFLAGS) -if test "" = "$CXXFLAGS"; then - CXXFLAGS="-g -O4" -fi - -AC_PROG_CXX -AC_PROG_LIBTOOL - -AC_CHECK_PROG(ERB, erb, erb) -if test "x$ERB" = x; then - AC_MSG_ERROR([cannot find erb. Ruby is needed to build.]) -fi - -AC_CHECK_LIB(stdc++, main) - -CXXFLAGS="-O4 -Wall $CXXFLAGS -I.." - -AC_OUTPUT([Makefile]) - diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp index b2a81ef..6027251 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/array.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/boolean.hpp b/cpp/type/boolean.hpp index 60f1714..86bd697 100644 --- a/cpp/type/boolean.hpp +++ b/cpp/type/boolean.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/float.hpp b/cpp/type/float.hpp index 2178434..108709d 100644 --- a/cpp/type/float.hpp +++ b/cpp/type/float.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/integer.hpp b/cpp/type/integer.hpp index 5cd10f6..ecb7b89 100644 --- a/cpp/type/integer.hpp +++ b/cpp/type/integer.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index 4585d66..c136d53 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/nil.hpp b/cpp/type/nil.hpp index ab0c363..93e66ff 100644 --- a/cpp/type/nil.hpp +++ b/cpp/type/nil.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/raw.hpp b/cpp/type/raw.hpp index b102ee8..b6ace3f 100644 --- a/cpp/type/raw.hpp +++ b/cpp/type/raw.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index a20f5d9..586d84c 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp deleted file mode 100644 index 0b9a476..0000000 --- a/cpp/unpack.cpp +++ /dev/null @@ -1,369 +0,0 @@ -// -// MessagePack for C++ deserializing routine -// -// Copyright (C) 2008-2009 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#include "msgpack/unpack.hpp" -#include "msgpack/unpack_define.h" -#include - -namespace msgpack { - - -//namespace { -struct unpack_user { - zone* z; - bool referenced; -}; -//} // noname namespace - - -#define msgpack_unpack_struct(name) \ - struct msgpack_unpacker ## name - -#define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker ## name - -#define msgpack_unpack_callback(name) \ - msgpack_unpack ## name - -#define msgpack_unpack_object object - -#define msgpack_unpack_user unpack_user - - -struct msgpack_unpacker_context; - -static void msgpack_unpacker_init(struct msgpack_unpacker_context* ctx); - -static object msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); - -static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, - const char* data, size_t len, size_t* off); - - -static inline object msgpack_unpack_init(unpack_user* u) -{ return object(); } - -static inline object msgpack_unpack_uint8(unpack_user* u, uint8_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_uint16(unpack_user* u, uint16_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_uint32(unpack_user* u, uint32_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_uint64(unpack_user* u, uint64_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_int8(unpack_user* u, int8_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_int16(unpack_user* u, int16_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_int32(unpack_user* u, int32_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_int64(unpack_user* u, int64_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_float(unpack_user* u, float d) -{ object o; o.type = type::DOUBLE; o.via.dec = d; return o; } - -static inline object msgpack_unpack_double(unpack_user* u, double d) -{ object o; o.type = type::DOUBLE; o.via.dec = d; return o; } - -static inline object msgpack_unpack_nil(unpack_user* u) -{ object o; o.type = type::NIL; return o; } - -static inline object msgpack_unpack_true(unpack_user* u) -{ object o; o.type = type::BOOLEAN; o.via.boolean = true; return o; } - -static inline object msgpack_unpack_false(unpack_user* u) -{ object o; o.type = type::BOOLEAN; o.via.boolean = false; return o; } - -static inline object msgpack_unpack_array(unpack_user* u, unsigned int n) -{ - object o; - o.type = type::ARRAY; - o.via.array.size = 0; - o.via.array.ptr = (object*)u->z->malloc(n*sizeof(object)); - return o; -} - -static inline void msgpack_unpack_array_item(unpack_user* u, object* c, object o) -{ c->via.array.ptr[c->via.array.size++] = o; } - -static inline object msgpack_unpack_map(unpack_user* u, unsigned int n) -{ - object o; - o.type = type::MAP; - o.via.map.size = 0; - o.via.map.ptr = (object_kv*)u->z->malloc(n*sizeof(object_kv)); - return o; -} - -static inline void msgpack_unpack_map_item(unpack_user* u, object* c, object k, object v) -{ - c->via.map.ptr[c->via.map.size].key = k; - c->via.map.ptr[c->via.map.size].val = v; - ++c->via.map.size; -} - -static inline object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) -{ - object o; - o.type = type::RAW; - o.via.raw.ptr = p; - o.via.raw.size = l; - u->referenced = true; - return o; -} - -#include "msgpack/unpack_template.h" - - -namespace { -struct context { - context() - { - msgpack_unpacker_init(&m_ctx); - unpack_user u = {NULL, false}; - m_ctx.user = u; - } - - ~context() { } - - int execute(const char* data, size_t len, size_t* off) - { - return msgpack_unpacker_execute(&m_ctx, data, len, off); - } - - object data() - { - return msgpack_unpacker_data(&m_ctx); - } - - void reset() - { - zone* z = m_ctx.user.z; - msgpack_unpacker_init(&m_ctx); - unpack_user u = {z, false}; - m_ctx.user = u; - } - - void set_zone(zone* z) - { - m_ctx.user.z = z; - } - - bool is_referenced() const - { - return m_ctx.user.referenced; - } - -private: - msgpack_unpacker_context m_ctx; - zone* m_zone; - -private: - context(const context&); -}; - -static inline context* as_ctx(void* m) -{ - return reinterpret_cast(m); -} - - -static const size_t COUNTER_SIZE = sizeof(unsigned int); - -static inline void init_count(void* buffer) -{ - *(volatile unsigned int*)buffer = 1; -} - -static inline void decl_count(void* buffer) -{ - //if(--*(unsigned int*)buffer == 0) { - if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { - free(buffer); - } -} - -static inline void incr_count(void* buffer) -{ - //++*(unsigned int*)buffer; - __sync_add_and_fetch((unsigned int*)buffer, 1); -} - -static inline unsigned int get_count(void* buffer) -{ - return *(volatile unsigned int*)buffer; -} - -} // noname namespace - - -unpacker::unpacker(size_t initial_buffer_size) : - m_buffer(NULL), - m_used(0), - m_free(0), - m_off(0), - m_zone(new zone()), - m_ctx(new context()), - m_initial_buffer_size(initial_buffer_size) -{ - if(m_initial_buffer_size < COUNTER_SIZE) { - m_initial_buffer_size = COUNTER_SIZE; - } - - as_ctx(m_ctx)->set_zone(m_zone.get()); - - m_buffer = (char*)::malloc(m_initial_buffer_size); - if(!m_buffer) { throw std::bad_alloc(); } - init_count(m_buffer); - - m_used = COUNTER_SIZE; - m_free = m_initial_buffer_size - m_used; - m_off = COUNTER_SIZE; -} - - -unpacker::~unpacker() -{ - delete as_ctx(m_ctx); - decl_count(m_buffer); -} - -void unpacker::expand_buffer(size_t len) -{ - if(m_used == m_off && get_count(m_buffer) == 1 && - !as_ctx(m_ctx)->is_referenced()) { - // rewind buffer - m_free += m_used - COUNTER_SIZE; - m_used = COUNTER_SIZE; - m_off = COUNTER_SIZE; - if(m_free >= len) { return; } - } - - if(m_off == COUNTER_SIZE) { - size_t next_size = (m_used + m_free) * 2; - while(next_size < len + m_used) { next_size *= 2; } - - char* tmp = (char*)::realloc(m_buffer, next_size); - if(!tmp) { throw std::bad_alloc(); } - - m_buffer = tmp; - m_free = next_size - m_used; - - } else { - size_t next_size = m_initial_buffer_size; // include COUNTER_SIZE - size_t not_parsed = m_used - m_off; - while(next_size < len + not_parsed + COUNTER_SIZE) { next_size *= 2; } - - char* tmp = (char*)::malloc(next_size); - if(!tmp) { throw std::bad_alloc(); } - init_count(tmp); - - try { - m_zone->push_finalizer(decl_count, m_buffer); - } catch (...) { free(tmp); throw; } - - memcpy(tmp+COUNTER_SIZE, m_buffer+m_off, not_parsed); - - m_buffer = tmp; - m_used = not_parsed + COUNTER_SIZE; - m_free = next_size - m_used; - m_off = COUNTER_SIZE; - } -} - -bool unpacker::execute() -{ - int ret = as_ctx(m_ctx)->execute(m_buffer, m_used, &m_off); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret == 0) { - return false; - } else { - return true; - } -} - -zone* unpacker::release_zone() -{ - m_zone->push_finalizer(decl_count, m_buffer); - incr_count(m_buffer); - - //std::auto_ptr old(new zone()); - //m_zone.swap(old); - zone* n = new zone(); - std::auto_ptr old(m_zone.release()); - m_zone.reset(n); - - as_ctx(m_ctx)->set_zone(m_zone.get()); - - return old.release(); -} - -object unpacker::data() -{ - return as_ctx(m_ctx)->data(); -} - -void unpacker::reset() -{ - //if(!m_zone->empty()) { delete release_zone(); } - as_ctx(m_ctx)->reset(); -} - - -object unpacker::unpack(const char* data, size_t len, zone& z, size_t* off) -{ - context ctx; - ctx.set_zone(&z); - if(off) { - size_t noff = *off; - int ret = ctx.execute(data, len, &noff); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret == 0) { - throw unpack_error("insufficient bytes"); - } - *off = noff; - } else { - size_t noff = 0; - int ret = ctx.execute(data, len, &noff); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret == 0) { - throw unpack_error("insufficient bytes"); - } else if(noff < len) { - throw unpack_error("extra bytes"); - } - } - return ctx.data(); -} - - -} // namespace msgpack - diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index cde45e7..8c77726 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -18,6 +18,7 @@ #ifndef MSGPACK_UNPACK_HPP__ #define MSGPACK_UNPACK_HPP__ +#include "msgpack/unpack.h" #include "msgpack/object.hpp" #include "msgpack/zone.hpp" #include @@ -36,21 +37,21 @@ struct unpack_error : public std::runtime_error { }; -class unpacker { +class unpacker : public msgpack_unpacker { public: unpacker(size_t initial_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE); ~unpacker(); public: - /*! 1. reserve buffer. at least `len' bytes of capacity will be ready */ - void reserve_buffer(size_t len); + /*! 1. reserve buffer. at least `size' bytes of capacity will be ready */ + void reserve_buffer(size_t size); /*! 2. read data to the buffer() up to buffer_capacity() bytes */ char* buffer(); size_t buffer_capacity() const; /*! 3. specify the number of bytes actually copied */ - void buffer_consumed(size_t len); + void buffer_consumed(size_t size); /*! 4. repeat execute() until it retunrs false */ bool execute(); @@ -114,71 +115,157 @@ public: size_t nonparsed_size() const; /*! skip specified size of non-parsed buffer, leaving the buffer */ - // Note that the `len' argument must be smaller than nonparsed_size() - void skip_nonparsed_buffer(size_t len); + // Note that the `size' argument must be smaller than nonparsed_size() + void skip_nonparsed_buffer(size_t size); /*! remove unparsed buffer from unpacker */ // Note that reset() leaves non-parsed buffer. void remove_nonparsed_buffer(); -private: - char* m_buffer; - size_t m_used; - size_t m_free; - size_t m_off; - - std::auto_ptr m_zone; - - void* m_ctx; - - size_t m_initial_buffer_size; - -private: - void expand_buffer(size_t len); - private: unpacker(const unpacker&); - -public: - static object unpack(const char* data, size_t len, zone& z, size_t* off = NULL); }; -inline void unpacker::reserve_buffer(size_t len) +typedef enum { + MSGPACK_UNPACK_SUCCESS = 2, + MSGPACK_UNPACK_EXTRA_BYTES = 1, + MSGPACK_UNPACK_CONTINUE = 0, + MSGPACK_UNPACK_PARSE_ERROR = -1, +} unpack_return; + +static unpack_return unpack(const char* data, size_t len, size_t* off, + zone* z, object* result); + + +// obsolete +static object unpack(const char* data, size_t len, zone& z, size_t* off = NULL); + + +inline unpacker::unpacker(size_t initial_buffer_size) { - if(m_free >= len) { return; } - expand_buffer(len); + if(!msgpack_unpacker_init(this, initial_buffer_size)) { + throw std::bad_alloc(); + } +} + +inline unpacker::~unpacker() +{ + msgpack_unpacker_destroy(this); +} + +inline void unpacker::reserve_buffer(size_t size) +{ + if(!msgpack_unpacker_reserve_buffer(this, size)) { + throw std::bad_alloc(); + } } inline char* unpacker::buffer() - { return m_buffer + m_used; } +{ + return msgpack_unpacker_buffer(this); +} inline size_t unpacker::buffer_capacity() const - { return m_free; } - -inline void unpacker::buffer_consumed(size_t len) { - m_used += len; - m_free -= len; + return msgpack_unpacker_buffer_capacity(this); +} + +inline void unpacker::buffer_consumed(size_t size) +{ + return msgpack_unpacker_buffer_consumed(this, size); +} + + +inline bool unpacker::execute() +{ + int ret = msgpack_unpacker_execute(this); + if(ret < 0) { + throw unpack_error("parse error"); + } else if(ret == 0) { + return false; + } else { + return true; + } +} + +inline object unpacker::data() +{ + msgpack_object obj = msgpack_unpacker_data(this); + return *reinterpret_cast(&obj); +} + +inline zone* unpacker::release_zone() +{ + if(!msgpack_unpacker_flush_zone(this)) { + throw std::bad_alloc(); + } + + zone* r = new zone(); + + msgpack_zone old = *this->z; + *this->z = *z; + *z = old; + + return r; +} + +inline void unpacker::reset() +{ + msgpack_unpacker_reset(this); } inline char* unpacker::nonparsed_buffer() - { return m_buffer + m_off; } +{ + return buf + off; +} inline size_t unpacker::nonparsed_size() const - { return m_used - m_off; } +{ + return used - off; +} -inline void unpacker::skip_nonparsed_buffer(size_t len) - { m_off += len; } +inline void unpacker::skip_nonparsed_buffer(size_t size) +{ + off += size; +} inline void unpacker::remove_nonparsed_buffer() - { m_used = m_off; } - - -inline object unpack(const char* data, size_t len, zone& z, size_t* off = NULL) { - return unpacker::unpack(data, len, z, off); + used = off; +} + + +inline unpack_return unpack(const char* data, size_t len, size_t* off, + zone* z, object* result) +{ + return (unpack_return)msgpack_unpack(data, len, off, + z, reinterpret_cast(result)); +} + +inline object unpack(const char* data, size_t len, zone& z, size_t* off) +{ + object result; + + switch( msgpack::unpack(data, len, off, &z, &result) ) { + case MSGPACK_UNPACK_SUCCESS: + return result; + + case MSGPACK_UNPACK_EXTRA_BYTES: + if(off) { + return result; + } else { + throw unpack_error("extra bytes"); + } + + case MSGPACK_UNPACK_CONTINUE: + throw unpack_error("insufficient bytes"); + + case MSGPACK_UNPACK_PARSE_ERROR: + default: + throw unpack_error("parse error"); + } } diff --git a/cpp/zone.cpp b/cpp/zone.cpp deleted file mode 100644 index f765266..0000000 --- a/cpp/zone.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// -// MessagePack for C++ memory pool -// -// Copyright (C) 2008-2009 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#include "msgpack/zone.hpp" -#include - -namespace msgpack { - - -zone::zone(size_t chunk_size) : - m_chunk_size(chunk_size) -{ - chunk dummy = {0, NULL, NULL}; - m_chunk_array.push_back(dummy); -} - -zone::~zone() -{ - clear(); -} - -namespace { - template - struct zone_finalize { - void operator() (Private& f) { - (*f.func)(f.obj); - } - }; - - template - struct zone_free { - void operator() (Private& c) { - ::free(c.alloc); - } - }; -} - -void zone::clear() -{ - std::for_each(m_finalizers.rbegin(), m_finalizers.rend(), - zone_finalize()); - m_finalizers.clear(); - - std::for_each(m_chunk_array.begin(), m_chunk_array.end(), - zone_free()); - m_chunk_array.resize(1); - m_chunk_array[0].ptr = NULL; - m_chunk_array[0].free = 0; -} - -bool zone::empty() const -{ - return m_chunk_array.back().alloc == NULL && - m_finalizers.empty(); -} - -void* zone::malloc(size_t size) -{ - if(m_chunk_array.back().free > size) { - char* p = (char*)m_chunk_array.back().ptr; - m_chunk_array.back().ptr = p + size; - m_chunk_array.back().free -= size; - return p; - } - - size_t sz = m_chunk_size; - while(sz < size) { sz *= 2; } - - chunk dummy = {0, NULL, NULL}; - m_chunk_array.push_back(dummy); - - char* p = (char*)::malloc(sz); - if(!p) { - m_chunk_array.pop_back(); - throw std::bad_alloc(); - } - - m_chunk_array.back().free = sz - size; - m_chunk_array.back().ptr = p + size; - m_chunk_array.back().alloc = p; - return p; -} - - -} // namespace msgpack - diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index c0eb2e3..06cb9d3 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -19,17 +19,15 @@ #define MSGPACK_ZONE_HPP__ #include "msgpack/object.hpp" +#include "msgpack/zone.h" #include #include -#ifndef MSGPACK_ZONE_CHUNK_SIZE -#define MSGPACK_ZONE_CHUNK_SIZE 2048 -#endif <% GENERATION_LIMIT = 15 %> namespace msgpack { -class zone { +class zone : public msgpack_zone { public: zone(size_t chunk_size = MSGPACK_ZONE_CHUNK_SIZE); ~zone(); @@ -37,11 +35,7 @@ public: public: void* malloc(size_t size); - void push_finalizer(void (*func)(void*), void* obj); - - void clear(); - - bool empty() const; + void push_finalizer(void (*func)(void*), void* data); <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> @@ -49,35 +43,39 @@ public: <%}%> private: - struct chunk { - size_t free; - void* ptr; - void* alloc; - }; - - std::vector m_chunk_array; - - struct finalizer { - void (*func)(void*); - void* obj; - }; - - std::vector m_finalizers; - template static void object_destructor(void* obj); - size_t m_chunk_size; - private: zone(const zone&); }; -inline void zone::push_finalizer(void (*func)(void*), void* obj) + +inline zone::zone(size_t chunk_size) { - finalizer f = {func, obj}; - m_finalizers.push_back(f); + msgpack_zone_init(this, chunk_size); +} + +inline zone::~zone() +{ + msgpack_zone_destroy(this); +} + +inline void* zone::malloc(size_t size) +{ + void* ptr = msgpack_zone_malloc(this, size); + if(!ptr) { + throw std::bad_alloc(); + } + return ptr; +} + +inline void zone::push_finalizer(void (*func)(void*), void* data) +{ + if(!msgpack_zone_push_finalizer(this, func, data)) { + throw std::bad_alloc(); + } } template @@ -93,7 +91,7 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) void* x = malloc(sizeof(T)); push_finalizer(&zone::object_destructor, x); try { return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); } - catch (...) { m_finalizers.pop_back(); throw; } + catch (...) { --finalizer_array.tail; throw; } } <%}%> @@ -101,3 +99,4 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) #endif /* msgpack/zone.hpp */ +// vim: ft=cpp ts=4 sw=4 softtabstop=4 noexpandtab smarttab diff --git a/example/stream.cc b/example/stream.cc index 49927de..aef4ec8 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -10,6 +10,7 @@ class Server { public: Server(int sock) : m_sock(sock) { } + ~Server() { } typedef std::auto_ptr auto_zone; diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index a45369b..ff30955 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -49,7 +49,7 @@ msgpack_unpack_struct_decl(_stack) { }; msgpack_unpack_struct_decl(_context) { - msgpack_unpack_user user; // must be first + msgpack_unpack_user user; unsigned int cs; unsigned int trail; unsigned int top; @@ -63,12 +63,12 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; - ctx->stack[0].obj = msgpack_unpack_callback(_init)(&ctx->user); + ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); } -msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* unpacker) +msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) { - return (unpacker)->stack[0].obj; + return (ctx)->stack[0].obj; } @@ -92,16 +92,14 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c int ret; #define push_simple_value(func) \ - obj = msgpack_unpack_callback(func)(user); \ - /*printf("obj %d\n",obj);*/ \ + if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \ goto _push #define push_fixed_value(func, arg) \ - obj = msgpack_unpack_callback(func)(user, arg); \ - /*printf("obj %d\n",obj);*/ \ + if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \ goto _push #define push_variable_value(func, base, pos, len) \ - obj = msgpack_unpack_callback(func)(user, (const char*)base, (const char*)pos, len); \ - /*printf("obj %d\n",obj);*/ \ + if(msgpack_unpack_callback(func)(user, \ + (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ goto _push #define again_fixed_trail(_cs, trail_len) \ @@ -115,7 +113,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c goto _fixed_trail_again #define start_container(func, count_, ct_) \ - stack[top].obj = msgpack_unpack_callback(func)(user, count_); \ + if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ if((count_) == 0) { obj = stack[top].obj; goto _push; } \ if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ stack[top].ct = ct_; \ @@ -264,11 +262,13 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c case CS_ARRAY_16: start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); case CS_ARRAY_32: + /* FIXME security guard */ start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); case CS_MAP_16: start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); case CS_MAP_32: + /* FIXME security guard */ start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); default: @@ -281,7 +281,7 @@ _push: c = &stack[top-1]; switch(c->ct) { case CT_ARRAY_ITEM: - msgpack_unpack_callback(_array_item)(user, &c->obj, obj); + if(msgpack_unpack_callback(_array_item)(user, &c->obj, obj) < 0) { goto _failed; } if(--c->count == 0) { obj = c->obj; --top; @@ -294,7 +294,7 @@ _push: c->ct = CT_MAP_VALUE; goto _header_again; case CT_MAP_VALUE: - msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj); + if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } if(--c->count == 0) { obj = c->obj; --top; diff --git a/ruby/gem/README b/ruby/gem/README index 78fe6cb..859ae2b 100644 --- a/ruby/gem/README +++ b/ruby/gem/README @@ -24,6 +24,6 @@ == Copyright -Author:: frsyuki -Copyright:: Copyright (c) 2009 frsyuki +Author:: frsyuki +Copyright:: Copyright (c) 2008-2009 frsyuki License:: Apache License, Version 2.0 diff --git a/ruby/gengem.sh b/ruby/gengem.sh index be9d14b..6c07be3 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -1,5 +1,7 @@ #!/bin/sh +mkdir -p gem/ext +mkdir -p gem/msgpack cp extconf.rb gem/ext/ cp pack.c gem/ext/ cp pack.h gem/ext/ diff --git a/ruby/unpack.c b/ruby/unpack.c index 4650ed1..4de4955 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -21,99 +21,99 @@ typedef struct { int finished; - VALUE origstr; -} msgpack_unpack_context; + VALUE source; +} unpack_user; #define msgpack_unpack_struct(name) \ - struct msgpack_unpacker ## name + struct template ## name #define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker ## name + ret template ## name #define msgpack_unpack_callback(name) \ template_callback ## name #define msgpack_unpack_object VALUE -#define msgpack_unpack_user msgpack_unpack_context +#define msgpack_unpack_user unpack_user -struct msgpack_unpacker_context; -typedef struct msgpack_unpacker_context msgpack_unpacker; +struct template_context; +typedef struct template_context msgpack_unpack_t; -static void msgpack_unpacker_init(msgpack_unpacker* ctx); +static void template_init(msgpack_unpack_t* u); -static VALUE msgpack_unpacker_data(msgpack_unpacker* ctx); +static VALUE template_data(msgpack_unpack_t* u); -static int msgpack_unpacker_execute(msgpack_unpacker* ctx, +static int template_execute(msgpack_unpack_t* u, const char* data, size_t len, size_t* off); -static inline VALUE template_callback_init(msgpack_unpack_context* x) +static inline VALUE template_callback_root(unpack_user* u) { return Qnil; } -static inline VALUE template_callback_uint8(msgpack_unpack_context* x, uint8_t d) -{ return INT2FIX(d); } +static inline int template_callback_uint8(unpack_user* u, uint8_t d, VALUE* o) +{ *o = INT2FIX(d); return 0; } -static inline VALUE template_callback_uint16(msgpack_unpack_context* x, uint16_t d) -{ return INT2FIX(d); } +static inline int template_callback_uint16(unpack_user* u, uint16_t d, VALUE* o) +{ *o = INT2FIX(d); return 0; } -static inline VALUE template_callback_uint32(msgpack_unpack_context* x, uint32_t d) -{ return UINT2NUM(d); } +static inline int template_callback_uint32(unpack_user* u, uint32_t d, VALUE* o) +{ *o = UINT2NUM(d); return 0; } -static inline VALUE template_callback_uint64(msgpack_unpack_context* x, uint64_t d) -{ return rb_ull2inum(d); } +static inline int template_callback_uint64(unpack_user* u, uint64_t d, VALUE* o) +{ *o = rb_ull2inum(d); return 0; } -static inline VALUE template_callback_int8(msgpack_unpack_context* x, int8_t d) -{ return INT2FIX((long)d); } +static inline int template_callback_int8(unpack_user* u, int8_t d, VALUE* o) +{ *o = INT2FIX((long)d); return 0; } -static inline VALUE template_callback_int16(msgpack_unpack_context* x, int16_t d) -{ return INT2FIX((long)d); } +static inline int template_callback_int16(unpack_user* u, int16_t d, VALUE* o) +{ *o = INT2FIX((long)d); return 0; } -static inline VALUE template_callback_int32(msgpack_unpack_context* x, int32_t d) -{ return INT2NUM((long)d); } +static inline int template_callback_int32(unpack_user* u, int32_t d, VALUE* o) +{ *o = INT2NUM((long)d); return 0; } -static inline VALUE template_callback_int64(msgpack_unpack_context* x, int64_t d) -{ return rb_ll2inum(d); } +static inline int template_callback_int64(unpack_user* u, int64_t d, VALUE* o) +{ *o = rb_ll2inum(d); return 0; } -static inline VALUE template_callback_float(msgpack_unpack_context* x, float d) -{ return rb_float_new(d); } +static inline int template_callback_float(unpack_user* u, float d, VALUE* o) +{ *o = rb_float_new(d); return 0; } -static inline VALUE template_callback_double(msgpack_unpack_context* x, double d) -{ return rb_float_new(d); } +static inline int template_callback_double(unpack_user* u, double d, VALUE* o) +{ *o = rb_float_new(d); return 0; } -static inline VALUE template_callback_nil(msgpack_unpack_context* x) -{ return Qnil; } +static inline int template_callback_nil(unpack_user* u, VALUE* o) +{ *o = Qnil; return 0; } -static inline VALUE template_callback_true(msgpack_unpack_context* x) -{ return Qtrue; } +static inline int template_callback_true(unpack_user* u, VALUE* o) +{ *o = Qtrue; return 0; } -static inline VALUE template_callback_false(msgpack_unpack_context* x) -{ return Qfalse; } +static inline int template_callback_false(unpack_user* u, VALUE* o) +{ *o = Qfalse; return 0;} -static inline VALUE template_callback_array(msgpack_unpack_context* x, unsigned int n) -{ return rb_ary_new2(n); } +static inline int template_callback_array(unpack_user* u, unsigned int n, VALUE* o) +{ *o = rb_ary_new2(n); return 0; } -static inline void template_callback_array_item(msgpack_unpack_context* x, VALUE* c, VALUE o) -{ rb_ary_push(*c, o); } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] +static inline int template_callback_array_item(unpack_user* u, VALUE* c, VALUE o) +{ rb_ary_push(*c, o); return 0; } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] -static inline VALUE template_callback_map(msgpack_unpack_context* x, unsigned int n) -{ return rb_hash_new(); } +static inline int template_callback_map(unpack_user* u, unsigned int n, VALUE* o) +{ *o = rb_hash_new(); return 0; } -static inline void template_callback_map_item(msgpack_unpack_context* x, VALUE* c, VALUE k, VALUE v) -{ rb_hash_aset(*c, k, v); } +static inline int template_callback_map_item(unpack_user* u, VALUE* c, VALUE k, VALUE v) +{ rb_hash_aset(*c, k, v); return 0; } -static inline VALUE template_callback_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l) -{ return l == 0 ? rb_str_new(0,0) : rb_str_substr(x->origstr, p - b, l); } +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, VALUE* o) +{ *o = (l == 0) ? rb_str_new(0,0) : rb_str_substr(u->source, p - b, l); return 0; } #include "msgpack/unpack_template.h" #define UNPACKER(from, name) \ - msgpack_unpacker *name = NULL; \ - Data_Get_Struct(from, msgpack_unpacker, name); \ + msgpack_unpack_t *name = NULL; \ + Data_Get_Struct(from, msgpack_unpack_t, name); \ if(name == NULL) { \ rb_raise(rb_eArgError, "NULL found for " # name " when shouldn't be."); \ } @@ -132,7 +132,7 @@ static void MessagePack_Unpacker_free(void* data) if(data) { free(data); } } -static void MessagePack_Unpacker_mark(msgpack_unpacker *mp) +static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp) { unsigned int i; for(i=0; i < mp->top; ++i) { @@ -144,7 +144,7 @@ static void MessagePack_Unpacker_mark(msgpack_unpacker *mp) static VALUE MessagePack_Unpacker_alloc(VALUE klass) { VALUE obj; - msgpack_unpacker* mp = ALLOC_N(msgpack_unpacker, 1); + msgpack_unpack_t* mp = ALLOC_N(msgpack_unpack_t, 1); obj = Data_Wrap_Struct(klass, MessagePack_Unpacker_mark, MessagePack_Unpacker_free, mp); return obj; @@ -153,9 +153,9 @@ static VALUE MessagePack_Unpacker_alloc(VALUE klass) static VALUE MessagePack_Unpacker_reset(VALUE self) { UNPACKER(self, mp); - msgpack_unpacker_init(mp); - msgpack_unpack_context ctx = {0, Qnil}; - mp->user = ctx; + template_init(mp); + unpack_user u = {0, Qnil}; + mp->user = u; return self; } @@ -180,9 +180,9 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE args) rb_raise(eUnpackError, "offset is bigger than data buffer size."); } - mp->user.origstr = data; - ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); - mp->user.origstr = Qnil; + mp->user.source = data; + ret = template_execute(mp, dptr, (size_t)dlen, &from); + mp->user.source = Qnil; if(ret < 0) { rb_raise(eUnpackError, "parse error."); @@ -235,13 +235,13 @@ static VALUE MessagePack_Unpacker_finished_p(VALUE self) static VALUE MessagePack_Unpacker_data(VALUE self) { UNPACKER(self, mp); - return msgpack_unpacker_data(mp); + return template_data(mp); } static VALUE MessagePack_unpack_impl(VALUE args) { - msgpack_unpacker* mp = (msgpack_unpacker*)((VALUE*)args)[0]; + msgpack_unpack_t* mp = (msgpack_unpack_t*)((VALUE*)args)[0]; VALUE data = ((VALUE*)args)[1]; size_t from = 0; @@ -249,9 +249,9 @@ static VALUE MessagePack_unpack_impl(VALUE args) long dlen = FIX2LONG(((VALUE*)args)[2]); int ret; - mp->user.origstr = data; - ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); - mp->user.origstr = Qnil; + mp->user.source = data; + ret = template_execute(mp, dptr, (size_t)dlen, &from); + mp->user.source = Qnil; if(ret < 0) { rb_raise(eUnpackError, "parse error."); @@ -261,7 +261,7 @@ static VALUE MessagePack_unpack_impl(VALUE args) if(from < dlen) { rb_raise(eUnpackError, "extra bytes."); } - return msgpack_unpacker_data(mp); + return template_data(mp); } } @@ -278,10 +278,10 @@ static VALUE MessagePack_unpack_rescue(VALUE args) static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) { CHECK_STRING_TYPE(data); - msgpack_unpacker mp; - msgpack_unpacker_init(&mp); - msgpack_unpack_context ctx = {0, Qnil}; - mp.user = ctx; + msgpack_unpack_t mp; + template_init(&mp); + unpack_user u = {0, Qnil}; + mp.user = u; rb_gc_disable(); VALUE args[3] = {(VALUE)&mp, data, limit}; From 11abec1093cf8c9914b5aeeac383d30abc1cad72 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 22 Feb 2009 15:14:21 +0900 Subject: [PATCH 0058/1172] c and c++: rewritten and integrated --- COPYING | 2 +- Makefile.am | 5 + README | 2 +- bootstrap | 8 +- c/AUTHORS | 1 - c/COPYING | 14 -- c/ChangeLog | 0 c/NEWS | 0 c/README | 21 --- c/bootstrap | 3 - c/configure.in | 16 -- c/object.c | 199 ++++++-------------- c/object.h | 23 +-- c/pack.h | 87 +++++---- c/unpack.c | 384 +++++++++++++++++++++++++++++++------- c/unpack.h | 103 ++++++---- c/zone.c | 253 +++++++++++++++++++------ c/zone.h | 49 ++++- configure.in | 38 +++- cpp/AUTHORS | 1 - cpp/COPYING | 14 -- cpp/ChangeLog | 0 cpp/Makefile.am | 7 +- cpp/NEWS | 0 cpp/README | 21 --- cpp/bootstrap | 3 - cpp/configure.in | 23 --- cpp/type/array.hpp | 2 +- cpp/type/boolean.hpp | 2 +- cpp/type/float.hpp | 2 +- cpp/type/integer.hpp | 2 +- cpp/type/map.hpp | 2 +- cpp/type/nil.hpp | 2 +- cpp/type/raw.hpp | 2 +- cpp/type/tuple.hpp.erb | 2 +- cpp/unpack.cpp | 369 ------------------------------------ cpp/unpack.hpp | 171 ++++++++++++----- cpp/zone.cpp | 100 ---------- cpp/zone.hpp.erb | 59 +++--- example/stream.cc | 1 + msgpack/unpack_template.h | 26 +-- ruby/gem/README | 4 +- ruby/gengem.sh | 2 + ruby/unpack.c | 134 ++++++------- 44 files changed, 1035 insertions(+), 1124 deletions(-) delete mode 100644 c/AUTHORS delete mode 100644 c/COPYING delete mode 100644 c/ChangeLog delete mode 100644 c/NEWS delete mode 100644 c/README delete mode 100755 c/bootstrap delete mode 100644 c/configure.in delete mode 100644 cpp/AUTHORS delete mode 100644 cpp/COPYING delete mode 100644 cpp/ChangeLog delete mode 100644 cpp/NEWS delete mode 100644 cpp/README delete mode 100755 cpp/bootstrap delete mode 100644 cpp/configure.in delete mode 100644 cpp/unpack.cpp delete mode 100644 cpp/zone.cpp diff --git a/COPYING b/COPYING index 5f100cd..6f5f220 100644 --- a/COPYING +++ b/COPYING @@ -1,4 +1,4 @@ -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Makefile.am b/Makefile.am index c23320a..3144972 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,9 @@ +if ENABLE_CXX +export ERB SUBDIRS = c cpp +else +SUBDIRS = c +endif nobase_include_HEADERS = \ msgpack/pack_define.h \ diff --git a/README b/README index 87d9279..fd5fa8d 100644 --- a/README +++ b/README @@ -54,7 +54,7 @@ Binary-based efficient data interchange format. -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/bootstrap b/bootstrap index 954fa89..eb16c35 100755 --- a/bootstrap +++ b/bootstrap @@ -31,10 +31,10 @@ if test x"$1" = x"--help"; then fi -if [ -z "$NO_NEST" ];then - cd c && ./bootstrap $@; cd .. - cd cpp && ./bootstrap $@; cd .. -fi +#if [ -z "$NO_NEST" ];then +# cd c && ./bootstrap $@; cd .. +# cd cpp && ./bootstrap $@; cd .. +#fi ACLOCAL="aclocal" diff --git a/c/AUTHORS b/c/AUTHORS deleted file mode 100644 index ababacb..0000000 --- a/c/AUTHORS +++ /dev/null @@ -1 +0,0 @@ -FURUHASHI Sadayuki diff --git a/c/COPYING b/c/COPYING deleted file mode 100644 index 6f5f220..0000000 --- a/c/COPYING +++ /dev/null @@ -1,14 +0,0 @@ -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/c/ChangeLog b/c/ChangeLog deleted file mode 100644 index e69de29..0000000 diff --git a/c/NEWS b/c/NEWS deleted file mode 100644 index e69de29..0000000 diff --git a/c/README b/c/README deleted file mode 100644 index 76dc221..0000000 --- a/c/README +++ /dev/null @@ -1,21 +0,0 @@ -MessagePack for C ------------------ -MessagePack is a binary-based efficient data interchange format. - - - -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - diff --git a/c/bootstrap b/c/bootstrap deleted file mode 100755 index 7e61b82..0000000 --- a/c/bootstrap +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -NO_NEST=1 -. ../bootstrap diff --git a/c/configure.in b/c/configure.in deleted file mode 100644 index 47eb8aa..0000000 --- a/c/configure.in +++ /dev/null @@ -1,16 +0,0 @@ -AC_INIT(unpack.c) -AM_INIT_AUTOMAKE(msgpackc, 0.1.0) -AC_CONFIG_HEADER(config.h) - -AC_SUBST(CFLAGS) -if test "" = "$CFLAGS"; then - CFLAGS="-g -O4" -fi - -AC_PROG_CC -AC_PROG_LIBTOOL - -CFLAGS="-O4 -Wall $CFLAGS -I.." - -AC_OUTPUT([Makefile]) - diff --git a/c/object.c b/c/object.c index 6376438..871153d 100644 --- a/c/object.c +++ b/c/object.c @@ -15,153 +15,78 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "msgpack/unpack.h" -#include "msgpack/unpack_define.h" #include "msgpack/object.h" +#include +#include -typedef struct { - msgpack_zone* z; - bool referenced; - bool failed; -} unpack_user; - -#define msgpack_unpack_struct(name) \ - struct msgpack_unpacker ## name - -#define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker ## name - -#define msgpack_unpack_callback(name) \ - msgpack_unpack ## name - -#define msgpack_unpack_object msgpack_object - -#define msgpack_unpack_user unpack_user - - -struct msgpack_unpacker_context; - -static void msgpack_unpacker_init(struct msgpack_unpacker_context* ctx); - -static msgpack_object msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); - -static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, - const char* data, size_t len, size_t* off); - -static inline msgpack_object msgpack_unpack_init(unpack_user* u) -{ msgpack_object o; return o; } - -static inline msgpack_object msgpack_unpack_uint8(unpack_user* u, uint8_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_uint16(unpack_user* u, uint16_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_uint32(unpack_user* u, uint32_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_uint64(unpack_user* u, uint64_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_int8(unpack_user* u, int8_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_int16(unpack_user* u, int16_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_int32(unpack_user* u, int32_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_int64(unpack_user* u, int64_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_float(unpack_user* u, float d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_DOUBLE; o.via.dec = d; return o; } - -static inline msgpack_object msgpack_unpack_double(unpack_user* u, double d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_DOUBLE; o.via.dec = d; return o; } - -static inline msgpack_object msgpack_unpack_nil(unpack_user* u) -{ msgpack_object o; o.type = MSGPACK_OBJECT_NIL; return o; } - -static inline msgpack_object msgpack_unpack_true(unpack_user* u) -{ msgpack_object o; o.type = MSGPACK_OBJECT_BOOLEAN; o.via.boolean = true; return o; } - -static inline msgpack_object msgpack_unpack_false(unpack_user* u) -{ msgpack_object o; o.type = MSGPACK_OBJECT_BOOLEAN; o.via.boolean = false; return o; } - -static inline msgpack_object msgpack_unpack_array(unpack_user* u, unsigned int n) +void msgpack_object_print(FILE* out, msgpack_object o) { - msgpack_object o; - o.type = MSGPACK_OBJECT_ARRAY; - o.via.array.size = 0; - o.via.array.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); - if(o.via.array.ptr == NULL) { u->failed = true; } - return o; -} + switch(o.type) { + case MSGPACK_OBJECT_NIL: + fprintf(out, "nil"); + break; -static inline void msgpack_unpack_array_item(unpack_user* u, msgpack_object* c, msgpack_object o) -{ - if(u->failed) { return; } - c->via.array.ptr[ c->via.array.size++ ] = o; -} + case MSGPACK_OBJECT_BOOLEAN: + fprintf(out, (o.via.boolean ? "true" : "false")); + break; -static inline msgpack_object msgpack_unpack_map(unpack_user* u, unsigned int n) -{ - msgpack_object o; - o.type = MSGPACK_OBJECT_MAP; - o.via.map.size = 0; - o.via.map.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object_kv)); - if(o.via.map.ptr == NULL) { u->failed = true; } - return o; -} + case MSGPACK_OBJECT_POSITIVE_INTEGER: + fprintf(out, "%"PRIu64, o.via.u64); + break; -static inline void msgpack_unpack_map_item(unpack_user* u, msgpack_object* c, msgpack_object k, msgpack_object v) -{ - if(u->failed) { return; } - c->via.map.ptr[c->via.map.size].key = k; - c->via.map.ptr[c->via.map.size].val = v; - ++c->via.map.size; -} + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + fprintf(out, "%"PRIi64, o.via.i64); + break; -static inline msgpack_object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) -{ - msgpack_object o; - o.type = MSGPACK_OBJECT_RAW; - o.via.raw.ptr = p; - o.via.raw.size = l; - u->referenced = true; - return o; -} + case MSGPACK_OBJECT_DOUBLE: + fprintf(out, "%f", o.via.dec); + break; -#include "msgpack/unpack_template.h" + case MSGPACK_OBJECT_RAW: + fprintf(out, "\""); + fwrite(o.via.raw.ptr, o.via.raw.size, 1, out); + fprintf(out, "\""); + break; -msgpack_object_unpack_return -msgpack_object_unpack(const char* data, size_t len, size_t* off, - msgpack_zone* z, msgpack_object* result) -{ - struct msgpack_unpacker_context ctx; - msgpack_unpacker_init(&ctx); - unpack_user u = {z, false, false}; - ctx.user = u; + case MSGPACK_OBJECT_ARRAY: + fprintf(out, "["); + if(o.via.array.size != 0) { + msgpack_object* p = o.via.array.ptr; + msgpack_object_print(out, *p); + ++p; + msgpack_object* const pend = o.via.array.ptr + o.via.array.size; + for(; p < pend; ++p) { + fprintf(out, ", "); + msgpack_object_print(out, *p); + } + } + fprintf(out, "]"); + break; + // FIXME loop optimiziation - size_t noff = (off ? *off : 0); - int ret = msgpack_unpacker_execute(&ctx, data, len, &noff); - if(ret < 0 || ctx.user.failed) { - return MSGPACK_OBJECT_PARSE_ERROR; - } else if(ret == 0) { - return MSGPACK_OBJECT_INSUFFICIENT_BYTES; + case MSGPACK_OBJECT_MAP: + fprintf(out, "{"); + if(o.via.map.size != 0) { + msgpack_object_kv* p = o.via.map.ptr; + msgpack_object_print(out, p->key); + fprintf(out, "=>"); + msgpack_object_print(out, p->val); + ++p; + msgpack_object_kv* const pend = o.via.map.ptr + o.via.map.size; + for(; p < pend; ++p) { + fprintf(out, ", "); + msgpack_object_print(out, p->key); + fprintf(out, "=>"); + msgpack_object_print(out, p->val); + } + } + fprintf(out, "}"); + break; + // FIXME loop optimiziation + + default: + // FIXME + fprintf(out, "#", o.type, o.via.u64); } - *result = msgpack_unpacker_data(&ctx); - if(off) { *off = noff; } - if(ret == 0) { - return MSGPACK_OBJECT_EXTRA_BYTES; - } - return MSGPACK_OBJECT_PARSE_SUCCESS; } - diff --git a/c/object.h b/c/object.h index b590ebe..7c603b3 100644 --- a/c/object.h +++ b/c/object.h @@ -22,6 +22,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -39,18 +40,18 @@ typedef enum { MSGPACK_OBJECT_MAP = 0x08, } msgpack_object_type; -struct _msgpack_object; -struct _msgpack_object_kv; +struct msgpack_object; +struct msgpack_object_kv; typedef struct { uint32_t size; - struct _msgpack_object* ptr; + struct msgpack_object* ptr; } msgpack_object_array; typedef struct { uint32_t size; - struct _msgpack_object_kv* ptr; + struct msgpack_object_kv* ptr; } msgpack_object_map; typedef struct { @@ -68,26 +69,18 @@ typedef union { msgpack_object_raw raw; } msgpack_object_union; -typedef struct _msgpack_object { +typedef struct msgpack_object { msgpack_object_type type; msgpack_object_union via; } msgpack_object; -typedef struct _msgpack_object_kv { +typedef struct msgpack_object_kv { msgpack_object key; msgpack_object val; } msgpack_object_kv; -typedef enum { - MSGPACK_OBJECT_PARSE_SUCCESS = 0, - MSGPACK_OBJECT_EXTRA_BYTES = 1, - MSGPACK_OBJECT_INSUFFICIENT_BYTES = -1, - MSGPACK_OBJECT_PARSE_ERROR = -2, -} msgpack_object_unpack_return; -msgpack_object_unpack_return -msgpack_object_unpack(const char* data, size_t len, size_t* off, - msgpack_zone* z, msgpack_object* result); +void msgpack_object_print(FILE* out, msgpack_object o); #ifdef __cplusplus diff --git a/c/pack.h b/c/pack.h index a510eff..46de722 100644 --- a/c/pack.h +++ b/c/pack.h @@ -28,50 +28,49 @@ extern "C" { #endif -typedef int (*msgpack_pack_write_t)(void* data, const char* buf, unsigned int len); +typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len); -typedef struct { +typedef struct msgpack_packer { void* data; - msgpack_pack_write_t callback; -} msgpack_pack_t; + msgpack_packer_write callback; +} msgpack_packer; -static void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback); +static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); -static msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback); +static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); +static void msgpack_packer_free(msgpack_packer* pk); -static void msgpack_pack_free(msgpack_pack_t* ctx); +static int msgpack_pack_short(msgpack_packer* pk, short d); +static int msgpack_pack_int(msgpack_packer* pk, int d); +static int msgpack_pack_long(msgpack_packer* pk, long d); +static int msgpack_pack_long_long(msgpack_packer* pk, long long d); +static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); +static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); +static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); +static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); -static int msgpack_pack_short(msgpack_pack_t* ctx, short d); -static int msgpack_pack_int(msgpack_pack_t* ctx, int d); -static int msgpack_pack_long(msgpack_pack_t* ctx, long d); -static int msgpack_pack_long_long(msgpack_pack_t* ctx, long long d); -static int msgpack_pack_unsigned_short(msgpack_pack_t* ctx, unsigned short d); -static int msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); -static int msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); -static int msgpack_pack_unsigned_long_long(msgpack_pack_t* ctx, unsigned long long d); +static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); +static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); +static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); +static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); +static int msgpack_pack_int8(msgpack_packer* pk, int8_t d); +static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); +static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); +static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); -static int msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); -static int msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); -static int msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); -static int msgpack_pack_uint64(msgpack_pack_t* ctx, uint64_t d); -static int msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); -static int msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); -static int msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); -static int msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); +static int msgpack_pack_float(msgpack_packer* pk, float d); +static int msgpack_pack_double(msgpack_packer* pk, double d); -static int msgpack_pack_float(msgpack_pack_t* ctx, float d); -static int msgpack_pack_double(msgpack_pack_t* ctx, double d); +static int msgpack_pack_nil(msgpack_packer* pk); +static int msgpack_pack_true(msgpack_packer* pk); +static int msgpack_pack_false(msgpack_packer* pk); -static int msgpack_pack_nil(msgpack_pack_t* ctx); -static int msgpack_pack_true(msgpack_pack_t* ctx); -static int msgpack_pack_false(msgpack_pack_t* ctx); +static int msgpack_pack_array(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); +static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); - -static int msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); -static int msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); +static int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); @@ -81,30 +80,30 @@ static int msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); #define msgpack_pack_inline_func_cint(name) \ inline int msgpack_pack ## name -#define msgpack_pack_user msgpack_pack_t* +#define msgpack_pack_user msgpack_packer* #define msgpack_pack_append_buffer(user, buf, len) \ return (*(user)->callback)((user)->data, (const char*)buf, len) #include "msgpack/pack_template.h" -inline void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback) +inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) { - ctx->data = data; - ctx->callback = callback; + pk->data = data; + pk->callback = callback; } -inline msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback) +inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback) { - msgpack_pack_t* ctx = (msgpack_pack_t*)calloc(1, sizeof(msgpack_pack_t)); - if(!ctx) { return NULL; } - msgpack_pack_init(ctx, data, callback); - return ctx; + msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer)); + if(!pk) { return NULL; } + msgpack_packer_init(pk, data, callback); + return pk; } -inline void msgpack_pack_free(msgpack_pack_t* ctx) +inline void msgpack_packer_free(msgpack_packer* pk) { - free(ctx); + free(pk); } diff --git a/c/unpack.c b/c/unpack.c index 03c67be..e06d97c 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -20,125 +20,371 @@ #include +typedef struct { + msgpack_zone* z; + bool* referenced; +} unpack_user; + + #define msgpack_unpack_struct(name) \ struct template ## name #define msgpack_unpack_func(ret, name) \ - ret template_func ## name + ret template ## name #define msgpack_unpack_callback(name) \ template_callback ## name -#define msgpack_unpack_object void* +#define msgpack_unpack_object msgpack_object -#define msgpack_unpack_user msgpack_unpack_t +#define msgpack_unpack_user unpack_user struct template_context; +typedef struct template_context template_context; -static void template_func_init(struct template_context* ctx); +static void template_init(template_context* ctx); -static void* template_func_data(struct template_context* ctx); +static msgpack_object template_data(template_context* ctx); -static int template_func_execute(struct template_context* ctx, +static int template_execute(template_context* ctx, const char* data, size_t len, size_t* off); -static inline void* template_callback_init(msgpack_unpack_t* x) -{ return NULL; } +static inline msgpack_object template_callback_root(unpack_user* u) +{ msgpack_object o; return o; } -static inline void* template_callback_uint8(msgpack_unpack_t* x, uint8_t d) -{ return x->callback.unpack_uint8(x->data, d); } +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_uint16(msgpack_unpack_t* x, uint16_t d) -{ return x->callback.unpack_uint16(x->data, d); } +static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_uint32(msgpack_unpack_t* x, uint32_t d) -{ return x->callback.unpack_uint32(x->data, d); } +static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_uint64(msgpack_unpack_t* x, uint64_t d) -{ return x->callback.unpack_uint64(x->data, d); } +static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_int8(msgpack_unpack_t* x, int8_t d) -{ return x->callback.unpack_int8(x->data, d); } +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_int16(msgpack_unpack_t* x, int16_t d) -{ return x->callback.unpack_int16(x->data, d); } +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_int32(msgpack_unpack_t* x, int32_t d) -{ return x->callback.unpack_int32(x->data, d); } +static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_int64(msgpack_unpack_t* x, int64_t d) -{ return x->callback.unpack_int64(x->data, d); } +static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_float(msgpack_unpack_t* x, float d) -{ return x->callback.unpack_float(x->data, d); } +static inline int template_callback_float(unpack_user* u, float d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_DOUBLE; o->via.dec = d; return 0; } -static inline void* template_callback_double(msgpack_unpack_t* x, double d) -{ return x->callback.unpack_double(x->data, d); } +static inline int template_callback_double(unpack_user* u, double d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_DOUBLE; o->via.dec = d; return 0; } -static inline void* template_callback_nil(msgpack_unpack_t* x) -{ return x->callback.unpack_nil(x->data); } +static inline int template_callback_nil(unpack_user* u, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_NIL; return 0; } -static inline void* template_callback_true(msgpack_unpack_t* x) -{ return x->callback.unpack_true(x->data); } +static inline int template_callback_true(unpack_user* u, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_BOOLEAN; o->via.boolean = true; return 0; } -static inline void* template_callback_false(msgpack_unpack_t* x) -{ return x->callback.unpack_false(x->data); } +static inline int template_callback_false(unpack_user* u, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_BOOLEAN; o->via.boolean = false; return 0; } -static inline void* template_callback_array(msgpack_unpack_t* x, unsigned int n) -{ return x->callback.unpack_array(x->data, n); } +static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_object* o) +{ + o->type = MSGPACK_OBJECT_ARRAY; + o->via.array.size = 0; + o->via.array.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); + if(o->via.array.ptr == NULL) { return -1; } + return 0; +} -static inline void template_callback_array_item(msgpack_unpack_t* x, void** c, void* o) -{ x->callback.unpack_array_item(x->data, *c, o); } +static inline int template_callback_array_item(unpack_user* u, msgpack_object* c, msgpack_object o) +{ c->via.array.ptr[c->via.array.size++] = o; return 0; } -static inline void* template_callback_map(msgpack_unpack_t* x, unsigned int n) -{ return x->callback.unpack_map(x->data, n); } +static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_object* o) +{ + o->type = MSGPACK_OBJECT_MAP; + o->via.map.size = 0; + o->via.map.ptr = (msgpack_object_kv*)msgpack_zone_malloc(u->z, n*sizeof(msgpack_object_kv)); + if(o->via.map.ptr == NULL) { return -1; } + return 0; +} -static inline void template_callback_map_item(msgpack_unpack_t* x, void** c, void* k, void* v) -{ x->callback.unpack_map_item(x->data, *c, k, v); } - -static inline void* template_callback_raw(msgpack_unpack_t* x, const char* b, const char* p, unsigned int l) -{ return x->callback.unpack_raw(x->data, b, p, l); } +static inline int template_callback_map_item(unpack_user* u, msgpack_object* c, msgpack_object k, msgpack_object v) +{ + c->via.map.ptr[c->via.map.size].key = k; + c->via.map.ptr[c->via.map.size].val = v; + ++c->via.map.size; + return 0; +} +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_object* o) +{ + o->type = MSGPACK_OBJECT_RAW; + o->via.raw.ptr = p; + o->via.raw.size = l; + *u->referenced = true; + return 0; +} #include "msgpack/unpack_template.h" -msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback) +#define CTX_CAST(m) ((template_context*)(m)) + + +static const size_t COUNTER_SIZE = sizeof(unsigned int); + +static inline void init_count(void* buf) { - struct template_context* ctx; - ctx = (struct template_context*)calloc(1, sizeof(struct template_context)); - if(ctx == NULL) { return NULL; } - template_func_init(ctx); - ((msgpack_unpack_t*)ctx)->data = data; - ((msgpack_unpack_t*)ctx)->callback = *callback; - return (msgpack_unpack_t*)ctx; + *(volatile unsigned int*)buf = 1; } -void msgpack_unpack_free(msgpack_unpack_t* ctx) +static inline void decl_count(void* buf) { - free((struct template_context*)ctx); + //if(--*(unsigned int*)buf == 0) { + if(__sync_sub_and_fetch((unsigned int*)buf, 1) == 0) { + free(buf); + } } -void* msgpack_unpack_data(msgpack_unpack_t* ctx) +static inline void incr_count(void* buf) { - return template_func_data((struct template_context*)ctx); + //++*(unsigned int*)buf; + __sync_add_and_fetch((unsigned int*)buf, 1); } -void msgpack_unpack_reset(msgpack_unpack_t* ctx) +static inline unsigned int get_count(void* buf) { - msgpack_unpack_t x = ((struct template_context*)ctx)->user; - template_func_init((struct template_context*)ctx); - ((struct template_context*)ctx)->user = x; -} - -int msgpack_unpack_execute(msgpack_unpack_t* ctx, - const char* data, size_t len, size_t* off) -{ - return template_func_execute( - (struct template_context*)ctx, - data, len, off); + return *(volatile unsigned int*)buf; } + +bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) +{ + if(initial_buffer_size < COUNTER_SIZE) { + initial_buffer_size = COUNTER_SIZE; + } + + char* buf = (char*)malloc(initial_buffer_size); + if(buf == NULL) { + return false; + } + + void* ctx = malloc(sizeof(template_context)); + if(ctx == NULL) { + free(buf); + return false; + } + + msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(z == NULL) { + free(ctx); + free(buf); + return false; + } + + mpac->buf = buf; + mpac->used = COUNTER_SIZE; + mpac->free = initial_buffer_size - mpac->used; + mpac->off = COUNTER_SIZE; + mpac->initial_buffer_size = initial_buffer_size; + mpac->z = z; + mpac->referenced = false; + mpac->ctx = ctx; + + init_count(mpac->buf); + + template_init(CTX_CAST(mpac->ctx)); + CTX_CAST(mpac->ctx)->user.z = mpac->z; + CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; + + return true; +} + +void msgpack_unpacker_destroy(msgpack_unpacker* mpac) +{ + msgpack_zone_free(mpac->z); + free(mpac->ctx); + decl_count(mpac->buf); +} + + +msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size) +{ + msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker)); + if(mpac == NULL) { + return NULL; + } + + if(!msgpack_unpacker_init(mpac, initial_buffer_size)) { + free(mpac); + return NULL; + } + + return mpac; +} + +void msgpack_unpacker_free(msgpack_unpacker* mpac) +{ + msgpack_unpacker_destroy(mpac); + free(mpac); +} + + +bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) +{ + if(mpac->used == mpac->off && get_count(mpac->buf) == 1 && !mpac->referenced) { + // rewind buffer + mpac->free += mpac->used - COUNTER_SIZE; + mpac->used = COUNTER_SIZE; + mpac->off = COUNTER_SIZE; + + if(mpac->free >= size) { + return true; + } + } + + if(mpac->off == COUNTER_SIZE) { + size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE + while(next_size < size + mpac->used) { + next_size *= 2; + } + + char* tmp = (char*)realloc(mpac->buf, next_size); + if(tmp == NULL) { + return false; + } + + mpac->buf = tmp; + mpac->free = next_size - mpac->used; + + } else { + size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE + size_t not_parsed = mpac->used - mpac->off; + while(next_size < size + not_parsed + COUNTER_SIZE) { + next_size *= 2; + } + + char* tmp = (char*)malloc(next_size); + if(tmp == NULL) { + return false; + } + + init_count(tmp); + + if(mpac->referenced) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + free(tmp); + return false; + } + mpac->referenced = false; + } else { + decl_count(mpac->buf); + } + + memcpy(tmp+COUNTER_SIZE, mpac->buf+mpac->off, not_parsed); + + mpac->buf = tmp; + mpac->used = not_parsed + COUNTER_SIZE; + mpac->free = next_size - mpac->used; + mpac->off = COUNTER_SIZE; + } + + return true; +} + +int msgpack_unpacker_execute(msgpack_unpacker* mpac) +{ + return template_execute(CTX_CAST(mpac->ctx), + mpac->buf, mpac->used, &mpac->off); +} + +msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) +{ + return template_data(CTX_CAST(mpac->ctx)); +} + +msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) +{ + if(!msgpack_unpacker_flush_zone(mpac)) { + return false; + } + + msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(z == NULL) { + return NULL; + } + + msgpack_zone* old = mpac->z; + mpac->z = z; + + return old; +} + +bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) +{ + if(mpac->referenced) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + return false; + } + mpac->referenced = false; + + incr_count(mpac->buf); + } + + return true; +} + +void msgpack_unpacker_reset(msgpack_unpacker* mpac) +{ + msgpack_zone* z = mpac->z; + template_init(CTX_CAST(mpac->ctx)); + CTX_CAST(mpac->ctx)->user.z = z; + CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; +} + + +msgpack_unpack_return +msgpack_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_object* result) +{ + template_context ctx; + template_init(&ctx); + + bool referenced = false; + ctx.user.z = z; + ctx.user.referenced = &referenced; + + size_t noff = 0; + if(off != NULL) { noff = *off; } + + int ret = template_execute(&ctx, data, len, &noff); + if(ret < 0) { + return MSGPACK_UNPACK_PARSE_ERROR; + } + + if(off != NULL) { *off = noff; } + + if(ret == 0) { + return MSGPACK_UNPACK_CONTINUE; + } + + *result = template_data(&ctx); + + if(noff < len) { + return MSGPACK_UNPACK_EXTRA_BYTES; + } + + return MSGPACK_UNPACK_SUCCESS; +} + diff --git a/c/unpack.h b/c/unpack.h index 4977f51..ab202a1 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -15,9 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MSGPACK_UNPACK_H__ -#define MSGPACK_UNPACK_H__ +#ifndef msgpack_unpacker_H__ +#define msgpack_unpacker_H__ +#include "msgpack/zone.h" +#include "msgpack/object.h" #include #include @@ -26,39 +28,76 @@ extern "C" { #endif -typedef struct { - void* (*unpack_uint8)(void* data, uint8_t d); - void* (*unpack_uint16)(void* data, uint16_t d); - void* (*unpack_uint32)(void* data, uint32_t d); - void* (*unpack_uint64)(void* data, uint64_t d); - void* (*unpack_int8)(void* data, int8_t d); - void* (*unpack_int16)(void* data, int16_t d); - void* (*unpack_int32)(void* data, int32_t d); - void* (*unpack_int64)(void* data, int64_t d); - void* (*unpack_float)(void* data, float d); - void* (*unpack_double)(void* data, double d); - void* (*unpack_nil)(void* data); - void* (*unpack_true)(void* data); - void* (*unpack_false)(void* data); - void* (*unpack_array)(void* data, unsigned int n); - void (*unpack_array_item)(void* data, void* c, void* o); - void* (*unpack_map)(void* data, unsigned int n); - void (*unpack_map_item)(void* data, void* c, void* k, void* v); - void* (*unpack_raw)(void* data, const char* b, const char* p, unsigned int l); -} msgpack_unpack_callback; +typedef struct msgpack_unpacker { + char* buf; + size_t used; + size_t free; + size_t off; + msgpack_zone* z; + bool referenced; + size_t initial_buffer_size; + void* ctx; +} msgpack_unpacker; -typedef struct { - void* data; - msgpack_unpack_callback callback; -} msgpack_unpack_t; -msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback); -void msgpack_unpack_free(msgpack_unpack_t* ctx); +bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size); +void msgpack_unpacker_destroy(msgpack_unpacker* mpac); -int msgpack_unpack_execute(msgpack_unpack_t* ctx, - const char* data, size_t len, size_t* off); -void* msgpack_unpack_data(msgpack_unpack_t* ctx); -void msgpack_unpack_reset(msgpack_unpack_t* ctx); +msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size); +void msgpack_unpacker_free(msgpack_unpacker* mpac); + +static inline bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size); +static inline char* msgpack_unpacker_buffer(msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac); +static inline void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size); + + +int msgpack_unpacker_execute(msgpack_unpacker* mpac); + +msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac); + +msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); + +void msgpack_unpacker_reset(msgpack_unpacker* mpac); + + +typedef enum { + MSGPACK_UNPACK_SUCCESS = 2, + MSGPACK_UNPACK_EXTRA_BYTES = 1, + MSGPACK_UNPACK_CONTINUE = 0, + MSGPACK_UNPACK_PARSE_ERROR = -1, +} msgpack_unpack_return; + +msgpack_unpack_return +msgpack_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_object* result); + + +bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); + +bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); + +bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) +{ + if(mpac->free >= size) { return true; } + return msgpack_unpacker_expand_buffer(mpac, size); +} + +char* msgpack_unpacker_buffer(msgpack_unpacker* mpac) +{ + return mpac->buf + mpac->used; +} + +size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac) +{ + return mpac->free; +} + +void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size) +{ + mpac->used += size; + mpac->free -= size; +} #ifdef __cplusplus diff --git a/c/zone.c b/c/zone.c index ccd702d..e891c82 100644 --- a/c/zone.c +++ b/c/zone.c @@ -19,85 +19,214 @@ #include #include -typedef struct { - size_t free; - void* ptr; - void* alloc; -} msgpack_zone_chunk; -struct _msgpack_zone { - msgpack_zone_chunk* array; - size_t ntail; - size_t usable; -}; - -msgpack_zone* msgpack_zone_new() +static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_size) { - return calloc(1, sizeof(msgpack_zone)); -} + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + const size_t nfirst = (sizeof(msgpack_zone_chunk) < 72/2) ? + 72 / sizeof(msgpack_zone_chunk) : 8; -void msgpack_zone_free(msgpack_zone* z) -{ - if(z->array) { - size_t i; - for(i=0; i <= z->ntail; ++i) { - free(z->array[i].alloc); - } + msgpack_zone_chunk* array = (msgpack_zone_chunk*)malloc( + sizeof(msgpack_zone_chunk) * nfirst); + if(!array) { + return false; } - free(z); + + const size_t sz = chunk_size; + + char* ptr = (char*)malloc(sz); + if(!ptr) { + free(array); + return NULL; + } + + ca->tail = array; + ca->end = array + nfirst; + ca->array = array; + + array[0].free = sz; + array[0].ptr = ptr; + array[0].alloc = ptr; + + return true; } - -void* msgpack_zone_malloc(msgpack_zone* z, size_t size) +static inline void destroy_chunk_array(msgpack_zone_chunk_array* ca) { - if(!z->array) { - const size_t n = (sizeof(msgpack_zone_chunk) < 72/2) ? - 72 / sizeof(msgpack_zone_chunk) : 8; - msgpack_zone_chunk* array = - (msgpack_zone_chunk*)malloc(sizeof(msgpack_zone_chunk) * n); - if(!array) { return NULL; } + msgpack_zone_chunk* chunk = ca->array; + for(; chunk != ca->tail+1; ++chunk) { + free(chunk->alloc); + } + free(ca->array); +} - size_t sz = 2048; /* FIXME chunk_size */ - while(sz < size) { sz *= 2; } - char* p = (char*)malloc(sz); - if(!p) { - free(array); +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +{ + msgpack_zone_chunk_array* const ca = &zone->chunk_array; + + msgpack_zone_chunk* chunk = ca->tail; + + if(chunk->free > size) { + // chunkã«ç©ºã容é‡ãŒã‚ã‚‹ + // 空ã容é‡ã‚’消費ã—ã¦è¿”ã™ + + char* ptr = chunk->ptr; + + chunk->ptr += size; + chunk->free -= size; + + return ptr; + } + + chunk = ++ca->tail; + + if(chunk == ca->end) { + // ca->arrayã«ç©ºããŒãªã„ + // ca->arrayã‚’æ‹¡å¼µã™ã‚‹ + + const size_t nused = ca->end - ca->array; + const size_t nnext = (ca->end - ca->array) * 2; + + chunk = (msgpack_zone_chunk*)realloc(ca->array, + sizeof(msgpack_zone_chunk) * nnext); + if(!chunk) { return NULL; } - z->array = array; - z->usable = n - 1; - array[0].free = sz - size; - array[0].ptr = p + size; - array[0].alloc = p; - return p; + ca->array = chunk; + ca->end = chunk + nnext; + chunk = ca->tail = chunk + nused; } - if(z->array[z->ntail].free > size) { - char* p = (char*)z->array[z->ntail].ptr; - z->array[z->ntail].ptr = p + size; - z->array[z->ntail].free -= size; - return p; + size_t sz = zone->chunk_size; + + while(sz < size) { + sz *= 2; } - if(z->usable <= z->ntail) { - const size_t n = (z->usable + 1) * 2; - msgpack_zone_chunk* tmp = - (msgpack_zone_chunk*)realloc(z->array, sizeof(msgpack_zone_chunk) * n); - if(!tmp) { return NULL; } - z->array = tmp; - z->usable = n - 1; + char* ptr = (char*)malloc(sz); + if(!ptr) { + return NULL; } - size_t sz = 2048; /* FIXME chunk_size */ - while(sz < size) { sz *= 2; } - char* p = (char*)malloc(sz); - if(!p) { return NULL; } + chunk->free = sz - size; + chunk->ptr = ptr + size; + chunk->alloc = ptr; - ++z->ntail; - z->array[z->ntail].free = sz - size; - z->array[z->ntail].ptr = p + size; - z->array[z->ntail].alloc = p; - return p; + return ptr; +} + + +static inline void init_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + fa->tail = NULL; + fa->end = NULL; + fa->array = NULL; +} + +static inline void destroy_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + // 逆順ã«å‘¼ã³å‡ºã— + msgpack_zone_finalizer* fin = fa->tail; + for(; fin != fa->array; --fin) { + (*(fin-1)->func)((fin-1)->data); + } + free(fa->array); +} + +bool msgpack_zone_push_finalizer(msgpack_zone* zone, + void (*func)(void* data), void* data) +{ + msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; + + msgpack_zone_finalizer* fin = fa->tail; + + if(fin == fa->end) { + // fa->arrayã«ç©ºããŒãªã„ + // fa->arrayã‚’æ‹¡å¼µã™ã‚‹ + + size_t nnext; + const size_t nused = fa->end - fa->array; + + if(nused == 0) { + // åˆå›žã®å‘¼ã³å‡ºã—：fa->tail == fa->end == fa->array == NULL + + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + nnext = (sizeof(msgpack_zone_finalizer) < 72/2) ? + 72 / sizeof(msgpack_zone_finalizer) : 8; + + } else { + nnext = (fa->end - fa->array) * 2; + } + + fin = (msgpack_zone_finalizer*)realloc(fa->array, + sizeof(msgpack_zone_finalizer) * nnext); + if(!fin) { + return false; + } + + fa->array = fin; + fa->end = fin + nnext; + fin = fa->tail = fin + nused; + } + + fin->func = func; + fin->data = data; + + ++fa->tail; + + return true; +} + + +bool msgpack_zone_is_empty(msgpack_zone* zone) +{ + msgpack_zone_chunk_array* const ca = &zone->chunk_array; + msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; + return ca->array[0].ptr == ca->array[0].alloc && + ca->tail == ca->array && + fa->tail == fa->array; +} + + +bool msgpack_zone_init(msgpack_zone* zone, size_t chunk_size) +{ + zone->chunk_size = chunk_size; + + if(!init_chunk_array(&zone->chunk_array, chunk_size)) { + return false; + } + + init_finalizer_array(&zone->finalizer_array); + + return true; +} + +void msgpack_zone_destroy(msgpack_zone* zone) +{ + destroy_finalizer_array(&zone->finalizer_array); + destroy_chunk_array(&zone->chunk_array); +} + + +msgpack_zone* msgpack_zone_new(size_t chunk_size) +{ + msgpack_zone* zone = (msgpack_zone*)malloc(sizeof(msgpack_zone)); + if(zone == NULL) { + return NULL; + } + + if(!msgpack_zone_init(zone, chunk_size)) { + free(zone); + return NULL; + } + + return zone; +} + +void msgpack_zone_free(msgpack_zone* zone) +{ + msgpack_zone_destroy(zone); + free(zone); } diff --git a/c/zone.h b/c/zone.h index ff6ca62..3dc9f52 100644 --- a/c/zone.h +++ b/c/zone.h @@ -19,19 +19,58 @@ #define MSGPACK_ZONE_H__ #include +#include #ifdef __cplusplus extern "C" { #endif -struct _msgpack_zone; -typedef struct _msgpack_zone msgpack_zone; +typedef struct msgpack_zone_chunk { + size_t free; + char* ptr; + void* alloc; +} msgpack_zone_chunk; -msgpack_zone* msgpack_zone_new(); -void msgpack_zone_free(msgpack_zone* z); +typedef struct msgpack_zone_finalizer { + void (*func)(void* data); + void* data; +} msgpack_zone_finalizer; -void* msgpack_zone_malloc(msgpack_zone* z, size_t size); +typedef struct msgpack_zone_chunk_array { + msgpack_zone_chunk* tail; + msgpack_zone_chunk* end; + msgpack_zone_chunk* array; +} msgpack_zone_chunk_array; + +typedef struct msgpack_zone_finalizer_array { + msgpack_zone_finalizer* tail; + msgpack_zone_finalizer* end; + msgpack_zone_finalizer* array; +} msgpack_zone_finalizer_array; + +typedef struct msgpack_zone { + msgpack_zone_chunk_array chunk_array; + msgpack_zone_finalizer_array finalizer_array; + size_t chunk_size; +} msgpack_zone; + +#ifndef MSGPACK_ZONE_CHUNK_SIZE +#define MSGPACK_ZONE_CHUNK_SIZE 2048 +#endif + +bool msgpack_zone_init(msgpack_zone* zone, size_t chunk_size); +void msgpack_zone_destroy(msgpack_zone* zone); + +msgpack_zone* msgpack_zone_new(size_t chunk_size); +void msgpack_zone_free(msgpack_zone* zone); + +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); + +bool msgpack_zone_push_finalizer(msgpack_zone* zone, + void (*func)(void* data), void* data); + +bool msgpack_zone_is_empty(msgpack_zone* zone); #ifdef __cplusplus diff --git a/configure.in b/configure.in index a464fb9..9fa6ac5 100644 --- a/configure.in +++ b/configure.in @@ -2,8 +2,42 @@ AC_INIT(msgpack/unpack_template.h) AM_INIT_AUTOMAKE(msgpack, 0.3.0) AC_CONFIG_HEADER(config.h) +AC_SUBST(CFLAGS) +if test "" = "$CFLAGS"; then + CFLAGS="-g -O4" +fi + +AC_PROG_CC + +CFLAGS="-O4 -Wall $CFLAGS -I.." + + +AC_MSG_CHECKING([if c++ api is enabled]) +AC_ARG_ENABLE(cxx, + AS_HELP_STRING([--disable-cxx], + [don't build c++ api.]) ) +AC_MSG_RESULT($enable_cxx) +if test "$enable_cxx" != "no"; then + AC_SUBST(CXXFLAGS) + if test "" = "$CXXFLAGS"; then + CXXFLAGS="-g -O4" + fi + + AC_CHECK_PROG(ERB, erb, erb) + if test "x$ERB" = x; then + AC_MSG_ERROR([cannot find erb. Ruby is needed to build.]) + fi +fi + +# FIXME +AC_PROG_CXX + +CXXFLAGS="-O4 -Wall $CXXFLAGS -I.." + + +AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no") + AC_PROG_LIBTOOL -AC_CONFIG_SUBDIRS([c cpp]) -AC_OUTPUT([Makefile]) +AC_OUTPUT([Makefile c/Makefile cpp/Makefile]) diff --git a/cpp/AUTHORS b/cpp/AUTHORS deleted file mode 100644 index ababacb..0000000 --- a/cpp/AUTHORS +++ /dev/null @@ -1 +0,0 @@ -FURUHASHI Sadayuki diff --git a/cpp/COPYING b/cpp/COPYING deleted file mode 100644 index 6f5f220..0000000 --- a/cpp/COPYING +++ /dev/null @@ -1,14 +0,0 @@ -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/cpp/ChangeLog b/cpp/ChangeLog deleted file mode 100644 index e69de29..0000000 diff --git a/cpp/Makefile.am b/cpp/Makefile.am index c7ddf4b..406c57b 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -1,9 +1,7 @@ lib_LTLIBRARIES = libmsgpack.la libmsgpack_la_SOURCES = \ - object.cpp \ - unpack.cpp \ - zone.cpp + object.cpp nobase_include_HEADERS = \ msgpack.hpp \ @@ -28,7 +26,6 @@ noinst_HEADERS = \ # FIXME object.lo: msgpack/type/tuple.hpp unpack.lo: msgpack/type/tuple.hpp msgpack/zone.hpp -zone.lo: msgpack/type/tuple.hpp msgpack/zone.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb $(ERB) $< > $@.tmp @@ -42,6 +39,8 @@ MOSTLYCLEANFILES = \ msgpack/type/tuple.hpp \ msgpack/zone.hpp +libmsgpack_la_LIBADD = -L../c -lmsgpackc + # -version-info CURRENT:REVISION:AGE libmsgpack_la_LDFLAGS = -version-info 1:0:0 diff --git a/cpp/NEWS b/cpp/NEWS deleted file mode 100644 index e69de29..0000000 diff --git a/cpp/README b/cpp/README deleted file mode 100644 index 96f18b1..0000000 --- a/cpp/README +++ /dev/null @@ -1,21 +0,0 @@ -MessagePack for C++ -------------------- -MessagePack is a binary-based efficient data interchange format. - - - -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - diff --git a/cpp/bootstrap b/cpp/bootstrap deleted file mode 100755 index 7e61b82..0000000 --- a/cpp/bootstrap +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -NO_NEST=1 -. ../bootstrap diff --git a/cpp/configure.in b/cpp/configure.in deleted file mode 100644 index 2c3e5d0..0000000 --- a/cpp/configure.in +++ /dev/null @@ -1,23 +0,0 @@ -AC_INIT(object.cpp) -AM_INIT_AUTOMAKE(msgpack, 0.1.0) -AC_CONFIG_HEADER(config.h) - -AC_SUBST(CXXFLAGS) -if test "" = "$CXXFLAGS"; then - CXXFLAGS="-g -O4" -fi - -AC_PROG_CXX -AC_PROG_LIBTOOL - -AC_CHECK_PROG(ERB, erb, erb) -if test "x$ERB" = x; then - AC_MSG_ERROR([cannot find erb. Ruby is needed to build.]) -fi - -AC_CHECK_LIB(stdc++, main) - -CXXFLAGS="-O4 -Wall $CXXFLAGS -I.." - -AC_OUTPUT([Makefile]) - diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp index b2a81ef..6027251 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/array.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/boolean.hpp b/cpp/type/boolean.hpp index 60f1714..86bd697 100644 --- a/cpp/type/boolean.hpp +++ b/cpp/type/boolean.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/float.hpp b/cpp/type/float.hpp index 2178434..108709d 100644 --- a/cpp/type/float.hpp +++ b/cpp/type/float.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/integer.hpp b/cpp/type/integer.hpp index 5cd10f6..ecb7b89 100644 --- a/cpp/type/integer.hpp +++ b/cpp/type/integer.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index 4585d66..c136d53 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/nil.hpp b/cpp/type/nil.hpp index ab0c363..93e66ff 100644 --- a/cpp/type/nil.hpp +++ b/cpp/type/nil.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/raw.hpp b/cpp/type/raw.hpp index b102ee8..b6ace3f 100644 --- a/cpp/type/raw.hpp +++ b/cpp/type/raw.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index a20f5d9..586d84c 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp deleted file mode 100644 index 0b9a476..0000000 --- a/cpp/unpack.cpp +++ /dev/null @@ -1,369 +0,0 @@ -// -// MessagePack for C++ deserializing routine -// -// Copyright (C) 2008-2009 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#include "msgpack/unpack.hpp" -#include "msgpack/unpack_define.h" -#include - -namespace msgpack { - - -//namespace { -struct unpack_user { - zone* z; - bool referenced; -}; -//} // noname namespace - - -#define msgpack_unpack_struct(name) \ - struct msgpack_unpacker ## name - -#define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker ## name - -#define msgpack_unpack_callback(name) \ - msgpack_unpack ## name - -#define msgpack_unpack_object object - -#define msgpack_unpack_user unpack_user - - -struct msgpack_unpacker_context; - -static void msgpack_unpacker_init(struct msgpack_unpacker_context* ctx); - -static object msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); - -static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, - const char* data, size_t len, size_t* off); - - -static inline object msgpack_unpack_init(unpack_user* u) -{ return object(); } - -static inline object msgpack_unpack_uint8(unpack_user* u, uint8_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_uint16(unpack_user* u, uint16_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_uint32(unpack_user* u, uint32_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_uint64(unpack_user* u, uint64_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_int8(unpack_user* u, int8_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_int16(unpack_user* u, int16_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_int32(unpack_user* u, int32_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_int64(unpack_user* u, int64_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_float(unpack_user* u, float d) -{ object o; o.type = type::DOUBLE; o.via.dec = d; return o; } - -static inline object msgpack_unpack_double(unpack_user* u, double d) -{ object o; o.type = type::DOUBLE; o.via.dec = d; return o; } - -static inline object msgpack_unpack_nil(unpack_user* u) -{ object o; o.type = type::NIL; return o; } - -static inline object msgpack_unpack_true(unpack_user* u) -{ object o; o.type = type::BOOLEAN; o.via.boolean = true; return o; } - -static inline object msgpack_unpack_false(unpack_user* u) -{ object o; o.type = type::BOOLEAN; o.via.boolean = false; return o; } - -static inline object msgpack_unpack_array(unpack_user* u, unsigned int n) -{ - object o; - o.type = type::ARRAY; - o.via.array.size = 0; - o.via.array.ptr = (object*)u->z->malloc(n*sizeof(object)); - return o; -} - -static inline void msgpack_unpack_array_item(unpack_user* u, object* c, object o) -{ c->via.array.ptr[c->via.array.size++] = o; } - -static inline object msgpack_unpack_map(unpack_user* u, unsigned int n) -{ - object o; - o.type = type::MAP; - o.via.map.size = 0; - o.via.map.ptr = (object_kv*)u->z->malloc(n*sizeof(object_kv)); - return o; -} - -static inline void msgpack_unpack_map_item(unpack_user* u, object* c, object k, object v) -{ - c->via.map.ptr[c->via.map.size].key = k; - c->via.map.ptr[c->via.map.size].val = v; - ++c->via.map.size; -} - -static inline object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) -{ - object o; - o.type = type::RAW; - o.via.raw.ptr = p; - o.via.raw.size = l; - u->referenced = true; - return o; -} - -#include "msgpack/unpack_template.h" - - -namespace { -struct context { - context() - { - msgpack_unpacker_init(&m_ctx); - unpack_user u = {NULL, false}; - m_ctx.user = u; - } - - ~context() { } - - int execute(const char* data, size_t len, size_t* off) - { - return msgpack_unpacker_execute(&m_ctx, data, len, off); - } - - object data() - { - return msgpack_unpacker_data(&m_ctx); - } - - void reset() - { - zone* z = m_ctx.user.z; - msgpack_unpacker_init(&m_ctx); - unpack_user u = {z, false}; - m_ctx.user = u; - } - - void set_zone(zone* z) - { - m_ctx.user.z = z; - } - - bool is_referenced() const - { - return m_ctx.user.referenced; - } - -private: - msgpack_unpacker_context m_ctx; - zone* m_zone; - -private: - context(const context&); -}; - -static inline context* as_ctx(void* m) -{ - return reinterpret_cast(m); -} - - -static const size_t COUNTER_SIZE = sizeof(unsigned int); - -static inline void init_count(void* buffer) -{ - *(volatile unsigned int*)buffer = 1; -} - -static inline void decl_count(void* buffer) -{ - //if(--*(unsigned int*)buffer == 0) { - if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { - free(buffer); - } -} - -static inline void incr_count(void* buffer) -{ - //++*(unsigned int*)buffer; - __sync_add_and_fetch((unsigned int*)buffer, 1); -} - -static inline unsigned int get_count(void* buffer) -{ - return *(volatile unsigned int*)buffer; -} - -} // noname namespace - - -unpacker::unpacker(size_t initial_buffer_size) : - m_buffer(NULL), - m_used(0), - m_free(0), - m_off(0), - m_zone(new zone()), - m_ctx(new context()), - m_initial_buffer_size(initial_buffer_size) -{ - if(m_initial_buffer_size < COUNTER_SIZE) { - m_initial_buffer_size = COUNTER_SIZE; - } - - as_ctx(m_ctx)->set_zone(m_zone.get()); - - m_buffer = (char*)::malloc(m_initial_buffer_size); - if(!m_buffer) { throw std::bad_alloc(); } - init_count(m_buffer); - - m_used = COUNTER_SIZE; - m_free = m_initial_buffer_size - m_used; - m_off = COUNTER_SIZE; -} - - -unpacker::~unpacker() -{ - delete as_ctx(m_ctx); - decl_count(m_buffer); -} - -void unpacker::expand_buffer(size_t len) -{ - if(m_used == m_off && get_count(m_buffer) == 1 && - !as_ctx(m_ctx)->is_referenced()) { - // rewind buffer - m_free += m_used - COUNTER_SIZE; - m_used = COUNTER_SIZE; - m_off = COUNTER_SIZE; - if(m_free >= len) { return; } - } - - if(m_off == COUNTER_SIZE) { - size_t next_size = (m_used + m_free) * 2; - while(next_size < len + m_used) { next_size *= 2; } - - char* tmp = (char*)::realloc(m_buffer, next_size); - if(!tmp) { throw std::bad_alloc(); } - - m_buffer = tmp; - m_free = next_size - m_used; - - } else { - size_t next_size = m_initial_buffer_size; // include COUNTER_SIZE - size_t not_parsed = m_used - m_off; - while(next_size < len + not_parsed + COUNTER_SIZE) { next_size *= 2; } - - char* tmp = (char*)::malloc(next_size); - if(!tmp) { throw std::bad_alloc(); } - init_count(tmp); - - try { - m_zone->push_finalizer(decl_count, m_buffer); - } catch (...) { free(tmp); throw; } - - memcpy(tmp+COUNTER_SIZE, m_buffer+m_off, not_parsed); - - m_buffer = tmp; - m_used = not_parsed + COUNTER_SIZE; - m_free = next_size - m_used; - m_off = COUNTER_SIZE; - } -} - -bool unpacker::execute() -{ - int ret = as_ctx(m_ctx)->execute(m_buffer, m_used, &m_off); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret == 0) { - return false; - } else { - return true; - } -} - -zone* unpacker::release_zone() -{ - m_zone->push_finalizer(decl_count, m_buffer); - incr_count(m_buffer); - - //std::auto_ptr old(new zone()); - //m_zone.swap(old); - zone* n = new zone(); - std::auto_ptr old(m_zone.release()); - m_zone.reset(n); - - as_ctx(m_ctx)->set_zone(m_zone.get()); - - return old.release(); -} - -object unpacker::data() -{ - return as_ctx(m_ctx)->data(); -} - -void unpacker::reset() -{ - //if(!m_zone->empty()) { delete release_zone(); } - as_ctx(m_ctx)->reset(); -} - - -object unpacker::unpack(const char* data, size_t len, zone& z, size_t* off) -{ - context ctx; - ctx.set_zone(&z); - if(off) { - size_t noff = *off; - int ret = ctx.execute(data, len, &noff); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret == 0) { - throw unpack_error("insufficient bytes"); - } - *off = noff; - } else { - size_t noff = 0; - int ret = ctx.execute(data, len, &noff); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret == 0) { - throw unpack_error("insufficient bytes"); - } else if(noff < len) { - throw unpack_error("extra bytes"); - } - } - return ctx.data(); -} - - -} // namespace msgpack - diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index cde45e7..8c77726 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -18,6 +18,7 @@ #ifndef MSGPACK_UNPACK_HPP__ #define MSGPACK_UNPACK_HPP__ +#include "msgpack/unpack.h" #include "msgpack/object.hpp" #include "msgpack/zone.hpp" #include @@ -36,21 +37,21 @@ struct unpack_error : public std::runtime_error { }; -class unpacker { +class unpacker : public msgpack_unpacker { public: unpacker(size_t initial_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE); ~unpacker(); public: - /*! 1. reserve buffer. at least `len' bytes of capacity will be ready */ - void reserve_buffer(size_t len); + /*! 1. reserve buffer. at least `size' bytes of capacity will be ready */ + void reserve_buffer(size_t size); /*! 2. read data to the buffer() up to buffer_capacity() bytes */ char* buffer(); size_t buffer_capacity() const; /*! 3. specify the number of bytes actually copied */ - void buffer_consumed(size_t len); + void buffer_consumed(size_t size); /*! 4. repeat execute() until it retunrs false */ bool execute(); @@ -114,71 +115,157 @@ public: size_t nonparsed_size() const; /*! skip specified size of non-parsed buffer, leaving the buffer */ - // Note that the `len' argument must be smaller than nonparsed_size() - void skip_nonparsed_buffer(size_t len); + // Note that the `size' argument must be smaller than nonparsed_size() + void skip_nonparsed_buffer(size_t size); /*! remove unparsed buffer from unpacker */ // Note that reset() leaves non-parsed buffer. void remove_nonparsed_buffer(); -private: - char* m_buffer; - size_t m_used; - size_t m_free; - size_t m_off; - - std::auto_ptr m_zone; - - void* m_ctx; - - size_t m_initial_buffer_size; - -private: - void expand_buffer(size_t len); - private: unpacker(const unpacker&); - -public: - static object unpack(const char* data, size_t len, zone& z, size_t* off = NULL); }; -inline void unpacker::reserve_buffer(size_t len) +typedef enum { + MSGPACK_UNPACK_SUCCESS = 2, + MSGPACK_UNPACK_EXTRA_BYTES = 1, + MSGPACK_UNPACK_CONTINUE = 0, + MSGPACK_UNPACK_PARSE_ERROR = -1, +} unpack_return; + +static unpack_return unpack(const char* data, size_t len, size_t* off, + zone* z, object* result); + + +// obsolete +static object unpack(const char* data, size_t len, zone& z, size_t* off = NULL); + + +inline unpacker::unpacker(size_t initial_buffer_size) { - if(m_free >= len) { return; } - expand_buffer(len); + if(!msgpack_unpacker_init(this, initial_buffer_size)) { + throw std::bad_alloc(); + } +} + +inline unpacker::~unpacker() +{ + msgpack_unpacker_destroy(this); +} + +inline void unpacker::reserve_buffer(size_t size) +{ + if(!msgpack_unpacker_reserve_buffer(this, size)) { + throw std::bad_alloc(); + } } inline char* unpacker::buffer() - { return m_buffer + m_used; } +{ + return msgpack_unpacker_buffer(this); +} inline size_t unpacker::buffer_capacity() const - { return m_free; } - -inline void unpacker::buffer_consumed(size_t len) { - m_used += len; - m_free -= len; + return msgpack_unpacker_buffer_capacity(this); +} + +inline void unpacker::buffer_consumed(size_t size) +{ + return msgpack_unpacker_buffer_consumed(this, size); +} + + +inline bool unpacker::execute() +{ + int ret = msgpack_unpacker_execute(this); + if(ret < 0) { + throw unpack_error("parse error"); + } else if(ret == 0) { + return false; + } else { + return true; + } +} + +inline object unpacker::data() +{ + msgpack_object obj = msgpack_unpacker_data(this); + return *reinterpret_cast(&obj); +} + +inline zone* unpacker::release_zone() +{ + if(!msgpack_unpacker_flush_zone(this)) { + throw std::bad_alloc(); + } + + zone* r = new zone(); + + msgpack_zone old = *this->z; + *this->z = *z; + *z = old; + + return r; +} + +inline void unpacker::reset() +{ + msgpack_unpacker_reset(this); } inline char* unpacker::nonparsed_buffer() - { return m_buffer + m_off; } +{ + return buf + off; +} inline size_t unpacker::nonparsed_size() const - { return m_used - m_off; } +{ + return used - off; +} -inline void unpacker::skip_nonparsed_buffer(size_t len) - { m_off += len; } +inline void unpacker::skip_nonparsed_buffer(size_t size) +{ + off += size; +} inline void unpacker::remove_nonparsed_buffer() - { m_used = m_off; } - - -inline object unpack(const char* data, size_t len, zone& z, size_t* off = NULL) { - return unpacker::unpack(data, len, z, off); + used = off; +} + + +inline unpack_return unpack(const char* data, size_t len, size_t* off, + zone* z, object* result) +{ + return (unpack_return)msgpack_unpack(data, len, off, + z, reinterpret_cast(result)); +} + +inline object unpack(const char* data, size_t len, zone& z, size_t* off) +{ + object result; + + switch( msgpack::unpack(data, len, off, &z, &result) ) { + case MSGPACK_UNPACK_SUCCESS: + return result; + + case MSGPACK_UNPACK_EXTRA_BYTES: + if(off) { + return result; + } else { + throw unpack_error("extra bytes"); + } + + case MSGPACK_UNPACK_CONTINUE: + throw unpack_error("insufficient bytes"); + + case MSGPACK_UNPACK_PARSE_ERROR: + default: + throw unpack_error("parse error"); + } } diff --git a/cpp/zone.cpp b/cpp/zone.cpp deleted file mode 100644 index f765266..0000000 --- a/cpp/zone.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// -// MessagePack for C++ memory pool -// -// Copyright (C) 2008-2009 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#include "msgpack/zone.hpp" -#include - -namespace msgpack { - - -zone::zone(size_t chunk_size) : - m_chunk_size(chunk_size) -{ - chunk dummy = {0, NULL, NULL}; - m_chunk_array.push_back(dummy); -} - -zone::~zone() -{ - clear(); -} - -namespace { - template - struct zone_finalize { - void operator() (Private& f) { - (*f.func)(f.obj); - } - }; - - template - struct zone_free { - void operator() (Private& c) { - ::free(c.alloc); - } - }; -} - -void zone::clear() -{ - std::for_each(m_finalizers.rbegin(), m_finalizers.rend(), - zone_finalize()); - m_finalizers.clear(); - - std::for_each(m_chunk_array.begin(), m_chunk_array.end(), - zone_free()); - m_chunk_array.resize(1); - m_chunk_array[0].ptr = NULL; - m_chunk_array[0].free = 0; -} - -bool zone::empty() const -{ - return m_chunk_array.back().alloc == NULL && - m_finalizers.empty(); -} - -void* zone::malloc(size_t size) -{ - if(m_chunk_array.back().free > size) { - char* p = (char*)m_chunk_array.back().ptr; - m_chunk_array.back().ptr = p + size; - m_chunk_array.back().free -= size; - return p; - } - - size_t sz = m_chunk_size; - while(sz < size) { sz *= 2; } - - chunk dummy = {0, NULL, NULL}; - m_chunk_array.push_back(dummy); - - char* p = (char*)::malloc(sz); - if(!p) { - m_chunk_array.pop_back(); - throw std::bad_alloc(); - } - - m_chunk_array.back().free = sz - size; - m_chunk_array.back().ptr = p + size; - m_chunk_array.back().alloc = p; - return p; -} - - -} // namespace msgpack - diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index c0eb2e3..06cb9d3 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -19,17 +19,15 @@ #define MSGPACK_ZONE_HPP__ #include "msgpack/object.hpp" +#include "msgpack/zone.h" #include #include -#ifndef MSGPACK_ZONE_CHUNK_SIZE -#define MSGPACK_ZONE_CHUNK_SIZE 2048 -#endif <% GENERATION_LIMIT = 15 %> namespace msgpack { -class zone { +class zone : public msgpack_zone { public: zone(size_t chunk_size = MSGPACK_ZONE_CHUNK_SIZE); ~zone(); @@ -37,11 +35,7 @@ public: public: void* malloc(size_t size); - void push_finalizer(void (*func)(void*), void* obj); - - void clear(); - - bool empty() const; + void push_finalizer(void (*func)(void*), void* data); <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> @@ -49,35 +43,39 @@ public: <%}%> private: - struct chunk { - size_t free; - void* ptr; - void* alloc; - }; - - std::vector m_chunk_array; - - struct finalizer { - void (*func)(void*); - void* obj; - }; - - std::vector m_finalizers; - template static void object_destructor(void* obj); - size_t m_chunk_size; - private: zone(const zone&); }; -inline void zone::push_finalizer(void (*func)(void*), void* obj) + +inline zone::zone(size_t chunk_size) { - finalizer f = {func, obj}; - m_finalizers.push_back(f); + msgpack_zone_init(this, chunk_size); +} + +inline zone::~zone() +{ + msgpack_zone_destroy(this); +} + +inline void* zone::malloc(size_t size) +{ + void* ptr = msgpack_zone_malloc(this, size); + if(!ptr) { + throw std::bad_alloc(); + } + return ptr; +} + +inline void zone::push_finalizer(void (*func)(void*), void* data) +{ + if(!msgpack_zone_push_finalizer(this, func, data)) { + throw std::bad_alloc(); + } } template @@ -93,7 +91,7 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) void* x = malloc(sizeof(T)); push_finalizer(&zone::object_destructor, x); try { return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); } - catch (...) { m_finalizers.pop_back(); throw; } + catch (...) { --finalizer_array.tail; throw; } } <%}%> @@ -101,3 +99,4 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) #endif /* msgpack/zone.hpp */ +// vim: ft=cpp ts=4 sw=4 softtabstop=4 noexpandtab smarttab diff --git a/example/stream.cc b/example/stream.cc index 49927de..aef4ec8 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -10,6 +10,7 @@ class Server { public: Server(int sock) : m_sock(sock) { } + ~Server() { } typedef std::auto_ptr auto_zone; diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index a45369b..ff30955 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -49,7 +49,7 @@ msgpack_unpack_struct_decl(_stack) { }; msgpack_unpack_struct_decl(_context) { - msgpack_unpack_user user; // must be first + msgpack_unpack_user user; unsigned int cs; unsigned int trail; unsigned int top; @@ -63,12 +63,12 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; - ctx->stack[0].obj = msgpack_unpack_callback(_init)(&ctx->user); + ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); } -msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* unpacker) +msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) { - return (unpacker)->stack[0].obj; + return (ctx)->stack[0].obj; } @@ -92,16 +92,14 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c int ret; #define push_simple_value(func) \ - obj = msgpack_unpack_callback(func)(user); \ - /*printf("obj %d\n",obj);*/ \ + if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \ goto _push #define push_fixed_value(func, arg) \ - obj = msgpack_unpack_callback(func)(user, arg); \ - /*printf("obj %d\n",obj);*/ \ + if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \ goto _push #define push_variable_value(func, base, pos, len) \ - obj = msgpack_unpack_callback(func)(user, (const char*)base, (const char*)pos, len); \ - /*printf("obj %d\n",obj);*/ \ + if(msgpack_unpack_callback(func)(user, \ + (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ goto _push #define again_fixed_trail(_cs, trail_len) \ @@ -115,7 +113,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c goto _fixed_trail_again #define start_container(func, count_, ct_) \ - stack[top].obj = msgpack_unpack_callback(func)(user, count_); \ + if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ if((count_) == 0) { obj = stack[top].obj; goto _push; } \ if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ stack[top].ct = ct_; \ @@ -264,11 +262,13 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c case CS_ARRAY_16: start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); case CS_ARRAY_32: + /* FIXME security guard */ start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); case CS_MAP_16: start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); case CS_MAP_32: + /* FIXME security guard */ start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); default: @@ -281,7 +281,7 @@ _push: c = &stack[top-1]; switch(c->ct) { case CT_ARRAY_ITEM: - msgpack_unpack_callback(_array_item)(user, &c->obj, obj); + if(msgpack_unpack_callback(_array_item)(user, &c->obj, obj) < 0) { goto _failed; } if(--c->count == 0) { obj = c->obj; --top; @@ -294,7 +294,7 @@ _push: c->ct = CT_MAP_VALUE; goto _header_again; case CT_MAP_VALUE: - msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj); + if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } if(--c->count == 0) { obj = c->obj; --top; diff --git a/ruby/gem/README b/ruby/gem/README index 78fe6cb..859ae2b 100644 --- a/ruby/gem/README +++ b/ruby/gem/README @@ -24,6 +24,6 @@ == Copyright -Author:: frsyuki -Copyright:: Copyright (c) 2009 frsyuki +Author:: frsyuki +Copyright:: Copyright (c) 2008-2009 frsyuki License:: Apache License, Version 2.0 diff --git a/ruby/gengem.sh b/ruby/gengem.sh index be9d14b..6c07be3 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -1,5 +1,7 @@ #!/bin/sh +mkdir -p gem/ext +mkdir -p gem/msgpack cp extconf.rb gem/ext/ cp pack.c gem/ext/ cp pack.h gem/ext/ diff --git a/ruby/unpack.c b/ruby/unpack.c index 4650ed1..4de4955 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -21,99 +21,99 @@ typedef struct { int finished; - VALUE origstr; -} msgpack_unpack_context; + VALUE source; +} unpack_user; #define msgpack_unpack_struct(name) \ - struct msgpack_unpacker ## name + struct template ## name #define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker ## name + ret template ## name #define msgpack_unpack_callback(name) \ template_callback ## name #define msgpack_unpack_object VALUE -#define msgpack_unpack_user msgpack_unpack_context +#define msgpack_unpack_user unpack_user -struct msgpack_unpacker_context; -typedef struct msgpack_unpacker_context msgpack_unpacker; +struct template_context; +typedef struct template_context msgpack_unpack_t; -static void msgpack_unpacker_init(msgpack_unpacker* ctx); +static void template_init(msgpack_unpack_t* u); -static VALUE msgpack_unpacker_data(msgpack_unpacker* ctx); +static VALUE template_data(msgpack_unpack_t* u); -static int msgpack_unpacker_execute(msgpack_unpacker* ctx, +static int template_execute(msgpack_unpack_t* u, const char* data, size_t len, size_t* off); -static inline VALUE template_callback_init(msgpack_unpack_context* x) +static inline VALUE template_callback_root(unpack_user* u) { return Qnil; } -static inline VALUE template_callback_uint8(msgpack_unpack_context* x, uint8_t d) -{ return INT2FIX(d); } +static inline int template_callback_uint8(unpack_user* u, uint8_t d, VALUE* o) +{ *o = INT2FIX(d); return 0; } -static inline VALUE template_callback_uint16(msgpack_unpack_context* x, uint16_t d) -{ return INT2FIX(d); } +static inline int template_callback_uint16(unpack_user* u, uint16_t d, VALUE* o) +{ *o = INT2FIX(d); return 0; } -static inline VALUE template_callback_uint32(msgpack_unpack_context* x, uint32_t d) -{ return UINT2NUM(d); } +static inline int template_callback_uint32(unpack_user* u, uint32_t d, VALUE* o) +{ *o = UINT2NUM(d); return 0; } -static inline VALUE template_callback_uint64(msgpack_unpack_context* x, uint64_t d) -{ return rb_ull2inum(d); } +static inline int template_callback_uint64(unpack_user* u, uint64_t d, VALUE* o) +{ *o = rb_ull2inum(d); return 0; } -static inline VALUE template_callback_int8(msgpack_unpack_context* x, int8_t d) -{ return INT2FIX((long)d); } +static inline int template_callback_int8(unpack_user* u, int8_t d, VALUE* o) +{ *o = INT2FIX((long)d); return 0; } -static inline VALUE template_callback_int16(msgpack_unpack_context* x, int16_t d) -{ return INT2FIX((long)d); } +static inline int template_callback_int16(unpack_user* u, int16_t d, VALUE* o) +{ *o = INT2FIX((long)d); return 0; } -static inline VALUE template_callback_int32(msgpack_unpack_context* x, int32_t d) -{ return INT2NUM((long)d); } +static inline int template_callback_int32(unpack_user* u, int32_t d, VALUE* o) +{ *o = INT2NUM((long)d); return 0; } -static inline VALUE template_callback_int64(msgpack_unpack_context* x, int64_t d) -{ return rb_ll2inum(d); } +static inline int template_callback_int64(unpack_user* u, int64_t d, VALUE* o) +{ *o = rb_ll2inum(d); return 0; } -static inline VALUE template_callback_float(msgpack_unpack_context* x, float d) -{ return rb_float_new(d); } +static inline int template_callback_float(unpack_user* u, float d, VALUE* o) +{ *o = rb_float_new(d); return 0; } -static inline VALUE template_callback_double(msgpack_unpack_context* x, double d) -{ return rb_float_new(d); } +static inline int template_callback_double(unpack_user* u, double d, VALUE* o) +{ *o = rb_float_new(d); return 0; } -static inline VALUE template_callback_nil(msgpack_unpack_context* x) -{ return Qnil; } +static inline int template_callback_nil(unpack_user* u, VALUE* o) +{ *o = Qnil; return 0; } -static inline VALUE template_callback_true(msgpack_unpack_context* x) -{ return Qtrue; } +static inline int template_callback_true(unpack_user* u, VALUE* o) +{ *o = Qtrue; return 0; } -static inline VALUE template_callback_false(msgpack_unpack_context* x) -{ return Qfalse; } +static inline int template_callback_false(unpack_user* u, VALUE* o) +{ *o = Qfalse; return 0;} -static inline VALUE template_callback_array(msgpack_unpack_context* x, unsigned int n) -{ return rb_ary_new2(n); } +static inline int template_callback_array(unpack_user* u, unsigned int n, VALUE* o) +{ *o = rb_ary_new2(n); return 0; } -static inline void template_callback_array_item(msgpack_unpack_context* x, VALUE* c, VALUE o) -{ rb_ary_push(*c, o); } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] +static inline int template_callback_array_item(unpack_user* u, VALUE* c, VALUE o) +{ rb_ary_push(*c, o); return 0; } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] -static inline VALUE template_callback_map(msgpack_unpack_context* x, unsigned int n) -{ return rb_hash_new(); } +static inline int template_callback_map(unpack_user* u, unsigned int n, VALUE* o) +{ *o = rb_hash_new(); return 0; } -static inline void template_callback_map_item(msgpack_unpack_context* x, VALUE* c, VALUE k, VALUE v) -{ rb_hash_aset(*c, k, v); } +static inline int template_callback_map_item(unpack_user* u, VALUE* c, VALUE k, VALUE v) +{ rb_hash_aset(*c, k, v); return 0; } -static inline VALUE template_callback_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l) -{ return l == 0 ? rb_str_new(0,0) : rb_str_substr(x->origstr, p - b, l); } +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, VALUE* o) +{ *o = (l == 0) ? rb_str_new(0,0) : rb_str_substr(u->source, p - b, l); return 0; } #include "msgpack/unpack_template.h" #define UNPACKER(from, name) \ - msgpack_unpacker *name = NULL; \ - Data_Get_Struct(from, msgpack_unpacker, name); \ + msgpack_unpack_t *name = NULL; \ + Data_Get_Struct(from, msgpack_unpack_t, name); \ if(name == NULL) { \ rb_raise(rb_eArgError, "NULL found for " # name " when shouldn't be."); \ } @@ -132,7 +132,7 @@ static void MessagePack_Unpacker_free(void* data) if(data) { free(data); } } -static void MessagePack_Unpacker_mark(msgpack_unpacker *mp) +static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp) { unsigned int i; for(i=0; i < mp->top; ++i) { @@ -144,7 +144,7 @@ static void MessagePack_Unpacker_mark(msgpack_unpacker *mp) static VALUE MessagePack_Unpacker_alloc(VALUE klass) { VALUE obj; - msgpack_unpacker* mp = ALLOC_N(msgpack_unpacker, 1); + msgpack_unpack_t* mp = ALLOC_N(msgpack_unpack_t, 1); obj = Data_Wrap_Struct(klass, MessagePack_Unpacker_mark, MessagePack_Unpacker_free, mp); return obj; @@ -153,9 +153,9 @@ static VALUE MessagePack_Unpacker_alloc(VALUE klass) static VALUE MessagePack_Unpacker_reset(VALUE self) { UNPACKER(self, mp); - msgpack_unpacker_init(mp); - msgpack_unpack_context ctx = {0, Qnil}; - mp->user = ctx; + template_init(mp); + unpack_user u = {0, Qnil}; + mp->user = u; return self; } @@ -180,9 +180,9 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE args) rb_raise(eUnpackError, "offset is bigger than data buffer size."); } - mp->user.origstr = data; - ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); - mp->user.origstr = Qnil; + mp->user.source = data; + ret = template_execute(mp, dptr, (size_t)dlen, &from); + mp->user.source = Qnil; if(ret < 0) { rb_raise(eUnpackError, "parse error."); @@ -235,13 +235,13 @@ static VALUE MessagePack_Unpacker_finished_p(VALUE self) static VALUE MessagePack_Unpacker_data(VALUE self) { UNPACKER(self, mp); - return msgpack_unpacker_data(mp); + return template_data(mp); } static VALUE MessagePack_unpack_impl(VALUE args) { - msgpack_unpacker* mp = (msgpack_unpacker*)((VALUE*)args)[0]; + msgpack_unpack_t* mp = (msgpack_unpack_t*)((VALUE*)args)[0]; VALUE data = ((VALUE*)args)[1]; size_t from = 0; @@ -249,9 +249,9 @@ static VALUE MessagePack_unpack_impl(VALUE args) long dlen = FIX2LONG(((VALUE*)args)[2]); int ret; - mp->user.origstr = data; - ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); - mp->user.origstr = Qnil; + mp->user.source = data; + ret = template_execute(mp, dptr, (size_t)dlen, &from); + mp->user.source = Qnil; if(ret < 0) { rb_raise(eUnpackError, "parse error."); @@ -261,7 +261,7 @@ static VALUE MessagePack_unpack_impl(VALUE args) if(from < dlen) { rb_raise(eUnpackError, "extra bytes."); } - return msgpack_unpacker_data(mp); + return template_data(mp); } } @@ -278,10 +278,10 @@ static VALUE MessagePack_unpack_rescue(VALUE args) static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) { CHECK_STRING_TYPE(data); - msgpack_unpacker mp; - msgpack_unpacker_init(&mp); - msgpack_unpack_context ctx = {0, Qnil}; - mp.user = ctx; + msgpack_unpack_t mp; + template_init(&mp); + unpack_user u = {0, Qnil}; + mp.user = u; rb_gc_disable(); VALUE args[3] = {(VALUE)&mp, data, limit}; From 96423688359d5211d11c12fe1c5d17ba01c760e6 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Sun, 22 Feb 2009 15:36:02 +0900 Subject: [PATCH 0059/1172] type conversion operator msgpack_object <-> msgpack::object --- configure.in | 2 +- cpp/object.hpp | 24 ++++++++++++++++++++++++ cpp/unpack.hpp | 3 +-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index 9fa6ac5..d7bb926 100644 --- a/configure.in +++ b/configure.in @@ -32,7 +32,7 @@ fi # FIXME AC_PROG_CXX -CXXFLAGS="-O4 -Wall $CXXFLAGS -I.." +CXXFLAGS="-O4 -Wall $CXXFLAGS -I.. -I../c" AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no") diff --git a/cpp/object.hpp b/cpp/object.hpp index 08e715d..c3f2872 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -18,8 +18,10 @@ #ifndef MSGPACK_OBJECT_HPP__ #define MSGPACK_OBJECT_HPP__ +#include "msgpack/object.h" #include "msgpack/pack.hpp" #include +#include #include #include #include @@ -72,6 +74,7 @@ struct object { object_array array; object_map map; object_raw raw; + object_raw ref; // obsolete }; type::object_type type; @@ -85,6 +88,10 @@ struct object { template void convert(T* v); + object(); + object(msgpack_object obj); + operator msgpack_object(); + private: struct implicit_type; @@ -192,6 +199,23 @@ inline bool operator!=(const object x, const object y) { return !(x == y); } +inline object::object() { } + +inline object::object(msgpack_object obj) +{ + // FIXME beter way? + ::memcpy(this, &obj, sizeof(obj)); +} + +inline object::operator msgpack_object() +{ + // FIXME beter way? + msgpack_object obj; + ::memcpy(&obj, this, sizeof(obj)); + return obj; +} + + inline object::implicit_type object::convert() { return implicit_type(*this); diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 8c77726..57623f2 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -191,8 +191,7 @@ inline bool unpacker::execute() inline object unpacker::data() { - msgpack_object obj = msgpack_unpacker_data(this); - return *reinterpret_cast(&obj); + return msgpack_unpacker_data(this); } inline zone* unpacker::release_zone() From d078eb0ad5b2b1c7749942a4618f6cfedd1067d5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 22 Feb 2009 15:36:02 +0900 Subject: [PATCH 0060/1172] type conversion operator msgpack_object <-> msgpack::object --- configure.in | 2 +- cpp/object.hpp | 24 ++++++++++++++++++++++++ cpp/unpack.hpp | 3 +-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index 9fa6ac5..d7bb926 100644 --- a/configure.in +++ b/configure.in @@ -32,7 +32,7 @@ fi # FIXME AC_PROG_CXX -CXXFLAGS="-O4 -Wall $CXXFLAGS -I.." +CXXFLAGS="-O4 -Wall $CXXFLAGS -I.. -I../c" AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no") diff --git a/cpp/object.hpp b/cpp/object.hpp index 08e715d..c3f2872 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -18,8 +18,10 @@ #ifndef MSGPACK_OBJECT_HPP__ #define MSGPACK_OBJECT_HPP__ +#include "msgpack/object.h" #include "msgpack/pack.hpp" #include +#include #include #include #include @@ -72,6 +74,7 @@ struct object { object_array array; object_map map; object_raw raw; + object_raw ref; // obsolete }; type::object_type type; @@ -85,6 +88,10 @@ struct object { template void convert(T* v); + object(); + object(msgpack_object obj); + operator msgpack_object(); + private: struct implicit_type; @@ -192,6 +199,23 @@ inline bool operator!=(const object x, const object y) { return !(x == y); } +inline object::object() { } + +inline object::object(msgpack_object obj) +{ + // FIXME beter way? + ::memcpy(this, &obj, sizeof(obj)); +} + +inline object::operator msgpack_object() +{ + // FIXME beter way? + msgpack_object obj; + ::memcpy(&obj, this, sizeof(obj)); + return obj; +} + + inline object::implicit_type object::convert() { return implicit_type(*this); diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 8c77726..57623f2 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -191,8 +191,7 @@ inline bool unpacker::execute() inline object unpacker::data() { - msgpack_object obj = msgpack_unpacker_data(this); - return *reinterpret_cast(&obj); + return msgpack_unpacker_data(this); } inline zone* unpacker::release_zone() From a0b82e39e11b4cfcbadc7e81b8927d77c1e1a1f5 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Sun, 22 Feb 2009 15:47:06 +0900 Subject: [PATCH 0061/1172] AC_CONFIG_AUX_DIR(ac) --- README | 2 +- bootstrap | 1 + configure.in | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README b/README index fd5fa8d..6e6ace4 100644 --- a/README +++ b/README @@ -9,7 +9,7 @@ Binary-based efficient data interchange format. UNIX-like platforms. Following programs is required to build: - - gcc >= 4.0 with C++ support + - gcc >= 4.1 with C++ support - ruby >= 1.8 (ruby is used as a preprocessor) diff --git a/bootstrap b/bootstrap index eb16c35..9ddb3da 100755 --- a/bootstrap +++ b/bootstrap @@ -35,6 +35,7 @@ fi # cd c && ./bootstrap $@; cd .. # cd cpp && ./bootstrap $@; cd .. #fi +mkdir -p ac ACLOCAL="aclocal" diff --git a/configure.in b/configure.in index d7bb926..2c1bee7 100644 --- a/configure.in +++ b/configure.in @@ -1,4 +1,5 @@ AC_INIT(msgpack/unpack_template.h) +AC_CONFIG_AUX_DIR(ac) AM_INIT_AUTOMAKE(msgpack, 0.3.0) AC_CONFIG_HEADER(config.h) From 0698304071f3cfb4f86135fcc0423609d8288d6b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 22 Feb 2009 15:47:06 +0900 Subject: [PATCH 0062/1172] AC_CONFIG_AUX_DIR(ac) --- README | 2 +- bootstrap | 1 + configure.in | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README b/README index fd5fa8d..6e6ace4 100644 --- a/README +++ b/README @@ -9,7 +9,7 @@ Binary-based efficient data interchange format. UNIX-like platforms. Following programs is required to build: - - gcc >= 4.0 with C++ support + - gcc >= 4.1 with C++ support - ruby >= 1.8 (ruby is used as a preprocessor) diff --git a/bootstrap b/bootstrap index eb16c35..9ddb3da 100755 --- a/bootstrap +++ b/bootstrap @@ -35,6 +35,7 @@ fi # cd c && ./bootstrap $@; cd .. # cd cpp && ./bootstrap $@; cd .. #fi +mkdir -p ac ACLOCAL="aclocal" diff --git a/configure.in b/configure.in index d7bb926..2c1bee7 100644 --- a/configure.in +++ b/configure.in @@ -1,4 +1,5 @@ AC_INIT(msgpack/unpack_template.h) +AC_CONFIG_AUX_DIR(ac) AM_INIT_AUTOMAKE(msgpack, 0.3.0) AC_CONFIG_HEADER(config.h) From 5d51129e65b54c32876aaf27db3a465ef89a6687 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Tue, 24 Feb 2009 16:37:01 +0900 Subject: [PATCH 0063/1172] add example/simple.c --- example/simple.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 example/simple.c diff --git a/example/simple.c b/example/simple.c new file mode 100644 index 0000000..df60424 --- /dev/null +++ b/example/simple.c @@ -0,0 +1,37 @@ +#include +#include + +int main(void) +{ + /* msgpack::sbuffer is a simple buffer implementation. */ + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + + /* serialize values into the buffer using msgpack_sbuffer_write callback function. */ + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + + msgpack_pack_array(&pk, 3); + msgpack_pack_int(&pk, 1); + msgpack_pack_true(&pk); + msgpack_pack_raw(&pk, 7); + msgpack_pack_raw_body(&pk, "example", 7); + + /* deserialize the buffer into msgpack_object instance. */ + /* deserialized object is valid during the msgpack_zone instance alive. */ + msgpack_zone mempool; + msgpack_zone_init(&mempool, 2048); + + msgpack_object deserialized; + msgpack_unpack(sbuf.ptr, sbuf.size, NULL, &mempool, &deserialized); + + /* print the deserialized object. */ + msgpack_object_print(stdout, deserialized); + puts(""); + + msgpack_zone_destroy(&mempool); + msgpack_sbuffer_destroy(&sbuf); + + return 0; +} + From aaaaecb8bad862afc66ad5a2772adb78b9082df8 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 24 Feb 2009 16:37:01 +0900 Subject: [PATCH 0064/1172] add example/simple.c --- example/simple.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 example/simple.c diff --git a/example/simple.c b/example/simple.c new file mode 100644 index 0000000..df60424 --- /dev/null +++ b/example/simple.c @@ -0,0 +1,37 @@ +#include +#include + +int main(void) +{ + /* msgpack::sbuffer is a simple buffer implementation. */ + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + + /* serialize values into the buffer using msgpack_sbuffer_write callback function. */ + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + + msgpack_pack_array(&pk, 3); + msgpack_pack_int(&pk, 1); + msgpack_pack_true(&pk); + msgpack_pack_raw(&pk, 7); + msgpack_pack_raw_body(&pk, "example", 7); + + /* deserialize the buffer into msgpack_object instance. */ + /* deserialized object is valid during the msgpack_zone instance alive. */ + msgpack_zone mempool; + msgpack_zone_init(&mempool, 2048); + + msgpack_object deserialized; + msgpack_unpack(sbuf.ptr, sbuf.size, NULL, &mempool, &deserialized); + + /* print the deserialized object. */ + msgpack_object_print(stdout, deserialized); + puts(""); + + msgpack_zone_destroy(&mempool); + msgpack_sbuffer_destroy(&sbuf); + + return 0; +} + From 4d708aa4da36759c5e2ca83e579d8f36e1c53fb0 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Tue, 24 Feb 2009 16:37:47 +0900 Subject: [PATCH 0065/1172] c: msgpack_pack_object --- c/object.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++ c/pack.h | 3 +++ cpp/msgpack.hpp | 1 + cpp/object.hpp | 32 +++++++++++----------- cpp/unpack.hpp | 1 + example/simple.cc | 24 ++++++++++------- example/stream.cc | 1 - 7 files changed, 106 insertions(+), 25 deletions(-) diff --git a/c/object.c b/c/object.c index 871153d..bcb2537 100644 --- a/c/object.c +++ b/c/object.c @@ -16,9 +16,78 @@ * limitations under the License. */ #include "msgpack/object.h" +#include "msgpack/pack.h" #include #include + +int msgpack_pack_object(msgpack_packer* pk, msgpack_object d) +{ + switch(d.type) { + case MSGPACK_OBJECT_NIL: + return msgpack_pack_nil(pk); + + case MSGPACK_OBJECT_BOOLEAN: + if(d.via.boolean) { + return msgpack_pack_true(pk); + } else { + return msgpack_pack_false(pk); + } + + case MSGPACK_OBJECT_POSITIVE_INTEGER: + return msgpack_pack_uint64(pk, d.via.u64); + + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + return msgpack_pack_int64(pk, d.via.i64); + + case MSGPACK_OBJECT_DOUBLE: + return msgpack_pack_double(pk, d.via.dec); + + case MSGPACK_OBJECT_RAW: + { + int ret = msgpack_pack_raw(pk, d.via.raw.size); + if(ret < 0) { return ret; } + return msgpack_pack_raw_body(pk, d.via.raw.ptr, d.via.raw.size); + } + + case MSGPACK_OBJECT_ARRAY: + { + int ret = msgpack_pack_array(pk, d.via.array.size); + if(ret < 0) { return ret; } + + msgpack_object* o = d.via.array.ptr; + msgpack_object* const oend = d.via.array.ptr + d.via.array.size; + for(; o != oend; ++o) { + ret = msgpack_pack_object(pk, *o); + if(ret < 0) { return ret; } + } + + return 0; + } + + case MSGPACK_OBJECT_MAP: + { + int ret = msgpack_pack_map(pk, d.via.map.size); + if(ret < 0) { return ret; } + + msgpack_object_kv* kv = d.via.map.ptr; + msgpack_object_kv* const kvend = d.via.map.ptr + d.via.map.size; + for(; kv != kvend; ++kv) { + ret = msgpack_pack_object(pk, kv->key); + if(ret < 0) { return ret; } + ret = msgpack_pack_object(pk, kv->val); + if(ret < 0) { return ret; } + } + + return 0; + } + + default: + return -1; + } +} + + void msgpack_object_print(FILE* out, msgpack_object o) { switch(o.type) { diff --git a/c/pack.h b/c/pack.h index 46de722..1a57ea4 100644 --- a/c/pack.h +++ b/c/pack.h @@ -22,6 +22,7 @@ #include #include #include "msgpack/pack_define.h" +#include "msgpack/object.h" #ifdef __cplusplus extern "C" { @@ -72,6 +73,8 @@ static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); static int msgpack_pack_raw(msgpack_packer* pk, size_t l); static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); +int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); + #define msgpack_pack_inline_func(name) \ diff --git a/cpp/msgpack.hpp b/cpp/msgpack.hpp index f00da06..58b40ac 100644 --- a/cpp/msgpack.hpp +++ b/cpp/msgpack.hpp @@ -20,3 +20,4 @@ #include "msgpack/pack.hpp" #include "msgpack/unpack.hpp" #include "msgpack/sbuffer.hpp" +#include "msgpack.h" diff --git a/cpp/object.hpp b/cpp/object.hpp index c3f2872..b4c21f1 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -162,18 +162,6 @@ inline packer& packer::pack(const T& v) return *this; } -template -inline void pack(Stream& s, const T& v) -{ - packer(s).pack(v); -} - -template -inline void pack_copy(packer& o, T v) -{ - pack(o, v); -} - inline object& operator>> (object o, object& v) { v = o; @@ -250,6 +238,20 @@ inline void pack(packer& o, const T& v) o.pack(v); } +// obsolete +template +inline void pack(Stream& s, const T& v) +{ + packer(s).pack(v); +} + +// obsolete +template +inline void pack_copy(packer& o, T v) +{ + pack(o, v); +} + template packer& operator<< (packer& o, const object& v) @@ -309,7 +311,7 @@ packer& operator<< (packer& o, const object& v) for(object* p(v.via.array.ptr), * const pend(v.via.array.ptr + v.via.array.size); p < pend; ++p) { - *p >> o; + o << *p; } return o; // FIXME loop optimiziation @@ -319,8 +321,8 @@ packer& operator<< (packer& o, const object& v) for(object_kv* p(v.via.map.ptr), * const pend(v.via.map.ptr + v.via.map.size); p < pend; ++p) { - p->key >> o; - p->val >> o; + o << p->key; + o << p->val; } return o; // FIXME loop optimiziation diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 57623f2..4e7ccf5 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -243,6 +243,7 @@ inline unpack_return unpack(const char* data, size_t len, size_t* off, z, reinterpret_cast(result)); } +// obsolete inline object unpack(const char* data, size_t len, zone& z, size_t* off) { object result; diff --git a/example/simple.cc b/example/simple.cc index 74beeae..55ecdf9 100644 --- a/example/simple.cc +++ b/example/simple.cc @@ -5,27 +5,33 @@ int main(void) { - // this is target object msgpack::type::tuple src(1, true, "example"); - // any classes that implements write(const char*,size_t) can be a buffer + // serialize the object into the buffer. + // any classes that implements write(const char*,size_t) can be a buffer. std::stringstream buffer; msgpack::pack(buffer, src); // send the buffer ... buffer.seekg(0); - // deserialize the buffer into msgpack::object type - msgpack::zone mempool; + // deserialize the buffer into msgpack::object instance. std::string str(buffer.str()); - msgpack::object deserialized = - msgpack::unpack(str.data(), str.size(), mempool); - // msgpack::object supports ostream + // deserialized object is valid during the msgpack::zone instance alive. + msgpack::zone mempool; + + msgpack::object deserialized; + msgpack::unpack(str.data(), str.size(), NULL, &mempool, &deserialized); + + // msgpack::object supports ostream. std::cout << deserialized << std::endl; - // convert msgpack::object type into the original type + // convert msgpack::object instance into the original type. + // if the type is mismatched, it throws msgpack::type_error exception. msgpack::type::tuple dst; - msgpack::convert(dst, deserialized); + deserialized.convert(&dst); + + return 0; } diff --git a/example/stream.cc b/example/stream.cc index aef4ec8..e57318f 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -84,7 +84,6 @@ struct fwriter { void write(const char* buf, size_t buflen) { size_t count = fwrite(buf, buflen, 1, m_fp); - //if(fwrite(buf, buflen, 1, m_fp) < 1) { if(count < 1) { std::cout << buflen << std::endl; std::cout << count << std::endl; From bdd13859b60d06ec6fabdbaccea81cbee189b8bc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 24 Feb 2009 16:37:47 +0900 Subject: [PATCH 0066/1172] c: msgpack_pack_object --- c/object.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++ c/pack.h | 3 +++ cpp/msgpack.hpp | 1 + cpp/object.hpp | 32 +++++++++++----------- cpp/unpack.hpp | 1 + example/simple.cc | 24 ++++++++++------- example/stream.cc | 1 - 7 files changed, 106 insertions(+), 25 deletions(-) diff --git a/c/object.c b/c/object.c index 871153d..bcb2537 100644 --- a/c/object.c +++ b/c/object.c @@ -16,9 +16,78 @@ * limitations under the License. */ #include "msgpack/object.h" +#include "msgpack/pack.h" #include #include + +int msgpack_pack_object(msgpack_packer* pk, msgpack_object d) +{ + switch(d.type) { + case MSGPACK_OBJECT_NIL: + return msgpack_pack_nil(pk); + + case MSGPACK_OBJECT_BOOLEAN: + if(d.via.boolean) { + return msgpack_pack_true(pk); + } else { + return msgpack_pack_false(pk); + } + + case MSGPACK_OBJECT_POSITIVE_INTEGER: + return msgpack_pack_uint64(pk, d.via.u64); + + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + return msgpack_pack_int64(pk, d.via.i64); + + case MSGPACK_OBJECT_DOUBLE: + return msgpack_pack_double(pk, d.via.dec); + + case MSGPACK_OBJECT_RAW: + { + int ret = msgpack_pack_raw(pk, d.via.raw.size); + if(ret < 0) { return ret; } + return msgpack_pack_raw_body(pk, d.via.raw.ptr, d.via.raw.size); + } + + case MSGPACK_OBJECT_ARRAY: + { + int ret = msgpack_pack_array(pk, d.via.array.size); + if(ret < 0) { return ret; } + + msgpack_object* o = d.via.array.ptr; + msgpack_object* const oend = d.via.array.ptr + d.via.array.size; + for(; o != oend; ++o) { + ret = msgpack_pack_object(pk, *o); + if(ret < 0) { return ret; } + } + + return 0; + } + + case MSGPACK_OBJECT_MAP: + { + int ret = msgpack_pack_map(pk, d.via.map.size); + if(ret < 0) { return ret; } + + msgpack_object_kv* kv = d.via.map.ptr; + msgpack_object_kv* const kvend = d.via.map.ptr + d.via.map.size; + for(; kv != kvend; ++kv) { + ret = msgpack_pack_object(pk, kv->key); + if(ret < 0) { return ret; } + ret = msgpack_pack_object(pk, kv->val); + if(ret < 0) { return ret; } + } + + return 0; + } + + default: + return -1; + } +} + + void msgpack_object_print(FILE* out, msgpack_object o) { switch(o.type) { diff --git a/c/pack.h b/c/pack.h index 46de722..1a57ea4 100644 --- a/c/pack.h +++ b/c/pack.h @@ -22,6 +22,7 @@ #include #include #include "msgpack/pack_define.h" +#include "msgpack/object.h" #ifdef __cplusplus extern "C" { @@ -72,6 +73,8 @@ static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); static int msgpack_pack_raw(msgpack_packer* pk, size_t l); static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); +int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); + #define msgpack_pack_inline_func(name) \ diff --git a/cpp/msgpack.hpp b/cpp/msgpack.hpp index f00da06..58b40ac 100644 --- a/cpp/msgpack.hpp +++ b/cpp/msgpack.hpp @@ -20,3 +20,4 @@ #include "msgpack/pack.hpp" #include "msgpack/unpack.hpp" #include "msgpack/sbuffer.hpp" +#include "msgpack.h" diff --git a/cpp/object.hpp b/cpp/object.hpp index c3f2872..b4c21f1 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -162,18 +162,6 @@ inline packer& packer::pack(const T& v) return *this; } -template -inline void pack(Stream& s, const T& v) -{ - packer(s).pack(v); -} - -template -inline void pack_copy(packer& o, T v) -{ - pack(o, v); -} - inline object& operator>> (object o, object& v) { v = o; @@ -250,6 +238,20 @@ inline void pack(packer& o, const T& v) o.pack(v); } +// obsolete +template +inline void pack(Stream& s, const T& v) +{ + packer(s).pack(v); +} + +// obsolete +template +inline void pack_copy(packer& o, T v) +{ + pack(o, v); +} + template packer& operator<< (packer& o, const object& v) @@ -309,7 +311,7 @@ packer& operator<< (packer& o, const object& v) for(object* p(v.via.array.ptr), * const pend(v.via.array.ptr + v.via.array.size); p < pend; ++p) { - *p >> o; + o << *p; } return o; // FIXME loop optimiziation @@ -319,8 +321,8 @@ packer& operator<< (packer& o, const object& v) for(object_kv* p(v.via.map.ptr), * const pend(v.via.map.ptr + v.via.map.size); p < pend; ++p) { - p->key >> o; - p->val >> o; + o << p->key; + o << p->val; } return o; // FIXME loop optimiziation diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 57623f2..4e7ccf5 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -243,6 +243,7 @@ inline unpack_return unpack(const char* data, size_t len, size_t* off, z, reinterpret_cast(result)); } +// obsolete inline object unpack(const char* data, size_t len, zone& z, size_t* off) { object result; diff --git a/example/simple.cc b/example/simple.cc index 74beeae..55ecdf9 100644 --- a/example/simple.cc +++ b/example/simple.cc @@ -5,27 +5,33 @@ int main(void) { - // this is target object msgpack::type::tuple src(1, true, "example"); - // any classes that implements write(const char*,size_t) can be a buffer + // serialize the object into the buffer. + // any classes that implements write(const char*,size_t) can be a buffer. std::stringstream buffer; msgpack::pack(buffer, src); // send the buffer ... buffer.seekg(0); - // deserialize the buffer into msgpack::object type - msgpack::zone mempool; + // deserialize the buffer into msgpack::object instance. std::string str(buffer.str()); - msgpack::object deserialized = - msgpack::unpack(str.data(), str.size(), mempool); - // msgpack::object supports ostream + // deserialized object is valid during the msgpack::zone instance alive. + msgpack::zone mempool; + + msgpack::object deserialized; + msgpack::unpack(str.data(), str.size(), NULL, &mempool, &deserialized); + + // msgpack::object supports ostream. std::cout << deserialized << std::endl; - // convert msgpack::object type into the original type + // convert msgpack::object instance into the original type. + // if the type is mismatched, it throws msgpack::type_error exception. msgpack::type::tuple dst; - msgpack::convert(dst, deserialized); + deserialized.convert(&dst); + + return 0; } diff --git a/example/stream.cc b/example/stream.cc index aef4ec8..e57318f 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -84,7 +84,6 @@ struct fwriter { void write(const char* buf, size_t buflen) { size_t count = fwrite(buf, buflen, 1, m_fp); - //if(fwrite(buf, buflen, 1, m_fp) < 1) { if(count < 1) { std::cout << buflen << std::endl; std::cout << count << std::endl; From 43360e378611ae550ece16554311e0de6e3cc046 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Wed, 25 Feb 2009 18:55:56 +0900 Subject: [PATCH 0067/1172] fix msgpack::unpack_return --- cpp/pack.hpp | 4 ++++ cpp/unpack.hpp | 16 ++++++++-------- example/stream.cc | 7 ++++--- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 52854ac..257ccb6 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -30,6 +30,7 @@ template class packer { public: packer(Stream& s); + ~packer(); public: template @@ -129,6 +130,9 @@ private: template packer::packer(Stream& s) : m_stream(s) { } +template +packer::~packer() { } + template inline packer& packer::pack_uint8(uint8_t d) { _pack_uint8(m_stream, d); return *this; } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 4e7ccf5..e6d35da 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -128,10 +128,10 @@ private: typedef enum { - MSGPACK_UNPACK_SUCCESS = 2, - MSGPACK_UNPACK_EXTRA_BYTES = 1, - MSGPACK_UNPACK_CONTINUE = 0, - MSGPACK_UNPACK_PARSE_ERROR = -1, + UNPACK_SUCCESS = 2, + UNPACK_EXTRA_BYTES = 1, + UNPACK_CONTINUE = 0, + UNPACK_PARSE_ERROR = -1, } unpack_return; static unpack_return unpack(const char* data, size_t len, size_t* off, @@ -249,20 +249,20 @@ inline object unpack(const char* data, size_t len, zone& z, size_t* off) object result; switch( msgpack::unpack(data, len, off, &z, &result) ) { - case MSGPACK_UNPACK_SUCCESS: + case UNPACK_SUCCESS: return result; - case MSGPACK_UNPACK_EXTRA_BYTES: + case UNPACK_EXTRA_BYTES: if(off) { return result; } else { throw unpack_error("extra bytes"); } - case MSGPACK_UNPACK_CONTINUE: + case UNPACK_CONTINUE: throw unpack_error("insufficient bytes"); - case MSGPACK_UNPACK_PARSE_ERROR: + case UNPACK_PARSE_ERROR: default: throw unpack_error("parse error"); } diff --git a/example/stream.cc b/example/stream.cc index e57318f..8fd92f9 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -113,6 +113,7 @@ int main(void) // client thread: fwriter writer(pair[1]); + msgpack::packer pk(writer); typedef msgpack::type::tuple put_t; typedef msgpack::type::tuple get_t; @@ -120,9 +121,9 @@ int main(void) put_t req1("put", "apple", "red"); put_t req2("put", "lemon", "yellow"); get_t req3("get", "apple"); - msgpack::pack(writer, req1); - msgpack::pack(writer, req2); - msgpack::pack(writer, req3); + pk.pack(req1); + pk.pack(req2); + pk.pack(req3); writer.flush(); writer.close(); From a89dfc70b997201cea3587de922924dec2bb28e7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 25 Feb 2009 18:55:56 +0900 Subject: [PATCH 0068/1172] fix msgpack::unpack_return --- cpp/pack.hpp | 4 ++++ cpp/unpack.hpp | 16 ++++++++-------- example/stream.cc | 7 ++++--- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 52854ac..257ccb6 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -30,6 +30,7 @@ template class packer { public: packer(Stream& s); + ~packer(); public: template @@ -129,6 +130,9 @@ private: template packer::packer(Stream& s) : m_stream(s) { } +template +packer::~packer() { } + template inline packer& packer::pack_uint8(uint8_t d) { _pack_uint8(m_stream, d); return *this; } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 4e7ccf5..e6d35da 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -128,10 +128,10 @@ private: typedef enum { - MSGPACK_UNPACK_SUCCESS = 2, - MSGPACK_UNPACK_EXTRA_BYTES = 1, - MSGPACK_UNPACK_CONTINUE = 0, - MSGPACK_UNPACK_PARSE_ERROR = -1, + UNPACK_SUCCESS = 2, + UNPACK_EXTRA_BYTES = 1, + UNPACK_CONTINUE = 0, + UNPACK_PARSE_ERROR = -1, } unpack_return; static unpack_return unpack(const char* data, size_t len, size_t* off, @@ -249,20 +249,20 @@ inline object unpack(const char* data, size_t len, zone& z, size_t* off) object result; switch( msgpack::unpack(data, len, off, &z, &result) ) { - case MSGPACK_UNPACK_SUCCESS: + case UNPACK_SUCCESS: return result; - case MSGPACK_UNPACK_EXTRA_BYTES: + case UNPACK_EXTRA_BYTES: if(off) { return result; } else { throw unpack_error("extra bytes"); } - case MSGPACK_UNPACK_CONTINUE: + case UNPACK_CONTINUE: throw unpack_error("insufficient bytes"); - case MSGPACK_UNPACK_PARSE_ERROR: + case UNPACK_PARSE_ERROR: default: throw unpack_error("parse error"); } diff --git a/example/stream.cc b/example/stream.cc index e57318f..8fd92f9 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -113,6 +113,7 @@ int main(void) // client thread: fwriter writer(pair[1]); + msgpack::packer pk(writer); typedef msgpack::type::tuple put_t; typedef msgpack::type::tuple get_t; @@ -120,9 +121,9 @@ int main(void) put_t req1("put", "apple", "red"); put_t req2("put", "lemon", "yellow"); get_t req3("get", "apple"); - msgpack::pack(writer, req1); - msgpack::pack(writer, req2); - msgpack::pack(writer, req3); + pk.pack(req1); + pk.pack(req2); + pk.pack(req3); writer.flush(); writer.close(); From 6ffee9e54abc54e84a2c81450bb1f607f0637a72 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Wed, 25 Feb 2009 23:31:53 +0900 Subject: [PATCH 0069/1172] c: msgpack_unpacker_buffered_size; c++: unpacker::buffered_size --- c/unpack.h | 6 ++++++ cpp/unpack.hpp | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/c/unpack.h b/c/unpack.h index ab202a1..46777b9 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -46,6 +46,7 @@ void msgpack_unpacker_destroy(msgpack_unpacker* mpac); msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size); void msgpack_unpacker_free(msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac); static inline bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size); static inline char* msgpack_unpacker_buffer(msgpack_unpacker* mpac); static inline size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac); @@ -77,6 +78,11 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); +size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac) +{ + return mpac->used; +} + bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) { if(mpac->free >= size) { return true; } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index e6d35da..6e9b641 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -43,6 +43,9 @@ public: ~unpacker(); public: + /*! 0. check if the buffered size is not exceed the assumption. */ + size_t buffered_size() const; + /*! 1. reserve buffer. at least `size' bytes of capacity will be ready */ void reserve_buffer(size_t size); @@ -154,6 +157,12 @@ inline unpacker::~unpacker() msgpack_unpacker_destroy(this); } + +inline size_t unpacker::buffered_size() const +{ + return msgpack_unpacker_buffered_size(this); +} + inline void unpacker::reserve_buffer(size_t size) { if(!msgpack_unpacker_reserve_buffer(this, size)) { From 8893523776e5258e2817711da75f00a6074a13ae Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 25 Feb 2009 23:31:53 +0900 Subject: [PATCH 0070/1172] c: msgpack_unpacker_buffered_size; c++: unpacker::buffered_size --- c/unpack.h | 6 ++++++ cpp/unpack.hpp | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/c/unpack.h b/c/unpack.h index ab202a1..46777b9 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -46,6 +46,7 @@ void msgpack_unpacker_destroy(msgpack_unpacker* mpac); msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size); void msgpack_unpacker_free(msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac); static inline bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size); static inline char* msgpack_unpacker_buffer(msgpack_unpacker* mpac); static inline size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac); @@ -77,6 +78,11 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); +size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac) +{ + return mpac->used; +} + bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) { if(mpac->free >= size) { return true; } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index e6d35da..6e9b641 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -43,6 +43,9 @@ public: ~unpacker(); public: + /*! 0. check if the buffered size is not exceed the assumption. */ + size_t buffered_size() const; + /*! 1. reserve buffer. at least `size' bytes of capacity will be ready */ void reserve_buffer(size_t size); @@ -154,6 +157,12 @@ inline unpacker::~unpacker() msgpack_unpacker_destroy(this); } + +inline size_t unpacker::buffered_size() const +{ + return msgpack_unpacker_buffered_size(this); +} + inline void unpacker::reserve_buffer(size_t size) { if(!msgpack_unpacker_reserve_buffer(this, size)) { From 65ef9cab66c39914da1292fb886c4c93c0a7312e Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Thu, 26 Feb 2009 01:15:14 +0900 Subject: [PATCH 0071/1172] remove msgpack_unpacker_buffered_size, add msgpack_unpacker_parsed_size --- c/unpack.c | 12 ++++++++---- c/unpack.h | 14 ++++++++------ cpp/unpack.hpp | 16 ++++++++-------- msgpack/unpack_template.h | 1 - ruby/unpack.c | 2 ++ 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/c/unpack.c b/c/unpack.c index e06d97c..865e0d4 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -197,6 +197,7 @@ bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) mpac->used = COUNTER_SIZE; mpac->free = initial_buffer_size - mpac->used; mpac->off = COUNTER_SIZE; + mpac->parsed = 0; mpac->initial_buffer_size = initial_buffer_size; mpac->z = z; mpac->referenced = false; @@ -305,8 +306,13 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) int msgpack_unpacker_execute(msgpack_unpacker* mpac) { - return template_execute(CTX_CAST(mpac->ctx), + size_t off = mpac->off; + int ret = template_execute(CTX_CAST(mpac->ctx), mpac->buf, mpac->used, &mpac->off); + if(mpac->off > off) { + mpac->parsed += mpac->off - off; + } + return ret; } msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) @@ -347,10 +353,8 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) void msgpack_unpacker_reset(msgpack_unpacker* mpac) { - msgpack_zone* z = mpac->z; template_init(CTX_CAST(mpac->ctx)); - CTX_CAST(mpac->ctx)->user.z = z; - CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; + mpac->parsed = 0; } diff --git a/c/unpack.h b/c/unpack.h index 46777b9..77fbd54 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -33,6 +33,7 @@ typedef struct msgpack_unpacker { size_t used; size_t free; size_t off; + size_t parsed; msgpack_zone* z; bool referenced; size_t initial_buffer_size; @@ -46,7 +47,6 @@ void msgpack_unpacker_destroy(msgpack_unpacker* mpac); msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size); void msgpack_unpacker_free(msgpack_unpacker* mpac); -static inline size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac); static inline bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size); static inline char* msgpack_unpacker_buffer(msgpack_unpacker* mpac); static inline size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac); @@ -61,6 +61,8 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); void msgpack_unpacker_reset(msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac); + typedef enum { MSGPACK_UNPACK_SUCCESS = 2, @@ -78,11 +80,6 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); -size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac) -{ - return mpac->used; -} - bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) { if(mpac->free >= size) { return true; } @@ -105,6 +102,11 @@ void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size) mpac->free -= size; } +size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac) +{ + return mpac->parsed; +} + #ifdef __cplusplus } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 6e9b641..0b7fe57 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -43,9 +43,6 @@ public: ~unpacker(); public: - /*! 0. check if the buffered size is not exceed the assumption. */ - size_t buffered_size() const; - /*! 1. reserve buffer. at least `size' bytes of capacity will be ready */ void reserve_buffer(size_t size); @@ -70,6 +67,9 @@ public: /*! 5.3. after release_zone(), re-initialize unpacker */ void reset(); + /*! 6. check if the parsed message size doesn't exceed assumption. */ + size_t parsed_size() const; + // Basic usage of the unpacker is as following: // @@ -158,11 +158,6 @@ inline unpacker::~unpacker() } -inline size_t unpacker::buffered_size() const -{ - return msgpack_unpacker_buffered_size(this); -} - inline void unpacker::reserve_buffer(size_t size) { if(!msgpack_unpacker_reserve_buffer(this, size)) { @@ -223,6 +218,11 @@ inline void unpacker::reset() msgpack_unpacker_reset(this); } +inline size_t unpacker::parsed_size() const +{ + return msgpack_unpacker_parsed_size(this); +} + inline char* unpacker::nonparsed_buffer() { diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index ff30955..d67fd1e 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -59,7 +59,6 @@ msgpack_unpack_struct_decl(_context) { msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) { - /*memset(ctx, 0, sizeof( msgpack_unpack_struct(_context) )); FIXME needed? */ ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; diff --git a/ruby/unpack.c b/ruby/unpack.c index 4de4955..411a94d 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -278,6 +278,7 @@ static VALUE MessagePack_unpack_rescue(VALUE args) static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) { CHECK_STRING_TYPE(data); + msgpack_unpack_t mp; template_init(&mp); unpack_user u = {0, Qnil}; @@ -288,6 +289,7 @@ static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) VALUE ret = rb_rescue(MessagePack_unpack_impl, (VALUE)args, MessagePack_unpack_rescue, Qnil); rb_gc_enable(); + return ret; } From 5697b9a15d89a397a53174b9eca2d96bc7ebaef0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 26 Feb 2009 01:15:14 +0900 Subject: [PATCH 0072/1172] remove msgpack_unpacker_buffered_size, add msgpack_unpacker_parsed_size --- c/unpack.c | 12 ++++++++---- c/unpack.h | 14 ++++++++------ cpp/unpack.hpp | 16 ++++++++-------- msgpack/unpack_template.h | 1 - ruby/unpack.c | 2 ++ 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/c/unpack.c b/c/unpack.c index e06d97c..865e0d4 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -197,6 +197,7 @@ bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) mpac->used = COUNTER_SIZE; mpac->free = initial_buffer_size - mpac->used; mpac->off = COUNTER_SIZE; + mpac->parsed = 0; mpac->initial_buffer_size = initial_buffer_size; mpac->z = z; mpac->referenced = false; @@ -305,8 +306,13 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) int msgpack_unpacker_execute(msgpack_unpacker* mpac) { - return template_execute(CTX_CAST(mpac->ctx), + size_t off = mpac->off; + int ret = template_execute(CTX_CAST(mpac->ctx), mpac->buf, mpac->used, &mpac->off); + if(mpac->off > off) { + mpac->parsed += mpac->off - off; + } + return ret; } msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) @@ -347,10 +353,8 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) void msgpack_unpacker_reset(msgpack_unpacker* mpac) { - msgpack_zone* z = mpac->z; template_init(CTX_CAST(mpac->ctx)); - CTX_CAST(mpac->ctx)->user.z = z; - CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; + mpac->parsed = 0; } diff --git a/c/unpack.h b/c/unpack.h index 46777b9..77fbd54 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -33,6 +33,7 @@ typedef struct msgpack_unpacker { size_t used; size_t free; size_t off; + size_t parsed; msgpack_zone* z; bool referenced; size_t initial_buffer_size; @@ -46,7 +47,6 @@ void msgpack_unpacker_destroy(msgpack_unpacker* mpac); msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size); void msgpack_unpacker_free(msgpack_unpacker* mpac); -static inline size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac); static inline bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size); static inline char* msgpack_unpacker_buffer(msgpack_unpacker* mpac); static inline size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac); @@ -61,6 +61,8 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); void msgpack_unpacker_reset(msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac); + typedef enum { MSGPACK_UNPACK_SUCCESS = 2, @@ -78,11 +80,6 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); -size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac) -{ - return mpac->used; -} - bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) { if(mpac->free >= size) { return true; } @@ -105,6 +102,11 @@ void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size) mpac->free -= size; } +size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac) +{ + return mpac->parsed; +} + #ifdef __cplusplus } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 6e9b641..0b7fe57 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -43,9 +43,6 @@ public: ~unpacker(); public: - /*! 0. check if the buffered size is not exceed the assumption. */ - size_t buffered_size() const; - /*! 1. reserve buffer. at least `size' bytes of capacity will be ready */ void reserve_buffer(size_t size); @@ -70,6 +67,9 @@ public: /*! 5.3. after release_zone(), re-initialize unpacker */ void reset(); + /*! 6. check if the parsed message size doesn't exceed assumption. */ + size_t parsed_size() const; + // Basic usage of the unpacker is as following: // @@ -158,11 +158,6 @@ inline unpacker::~unpacker() } -inline size_t unpacker::buffered_size() const -{ - return msgpack_unpacker_buffered_size(this); -} - inline void unpacker::reserve_buffer(size_t size) { if(!msgpack_unpacker_reserve_buffer(this, size)) { @@ -223,6 +218,11 @@ inline void unpacker::reset() msgpack_unpacker_reset(this); } +inline size_t unpacker::parsed_size() const +{ + return msgpack_unpacker_parsed_size(this); +} + inline char* unpacker::nonparsed_buffer() { diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index ff30955..d67fd1e 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -59,7 +59,6 @@ msgpack_unpack_struct_decl(_context) { msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) { - /*memset(ctx, 0, sizeof( msgpack_unpack_struct(_context) )); FIXME needed? */ ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; diff --git a/ruby/unpack.c b/ruby/unpack.c index 4de4955..411a94d 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -278,6 +278,7 @@ static VALUE MessagePack_unpack_rescue(VALUE args) static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) { CHECK_STRING_TYPE(data); + msgpack_unpack_t mp; template_init(&mp); unpack_user u = {0, Qnil}; @@ -288,6 +289,7 @@ static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) VALUE ret = rb_rescue(MessagePack_unpack_impl, (VALUE)args, MessagePack_unpack_rescue, Qnil); rb_gc_enable(); + return ret; } From c0baf9b87349a0ba250aa8ecf0df420db506dcdd Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Thu, 26 Feb 2009 01:27:00 +0900 Subject: [PATCH 0073/1172] add msgpack_unpacker_message_size --- c/unpack.h | 10 +++++++++- cpp/unpack.hpp | 12 +++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/c/unpack.h b/c/unpack.h index 77fbd54..779e256 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -61,7 +61,8 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); void msgpack_unpacker_reset(msgpack_unpacker* mpac); -static inline size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_message_size(const msgpack_unpacker* mpac); + typedef enum { @@ -76,6 +77,8 @@ msgpack_unpack(const char* data, size_t len, size_t* off, msgpack_zone* z, msgpack_object* result); +static inline size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac); + bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); @@ -102,6 +105,11 @@ void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size) mpac->free -= size; } +size_t msgpack_unpacker_message_size(const msgpack_unpacker* mpac) +{ + return mpac->parsed - mpac->off + mpac->used; +} + size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac) { return mpac->parsed; diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 0b7fe57..e140b36 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -67,8 +67,8 @@ public: /*! 5.3. after release_zone(), re-initialize unpacker */ void reset(); - /*! 6. check if the parsed message size doesn't exceed assumption. */ - size_t parsed_size() const; + /*! 6. check if the size of message doesn't exceed assumption. */ + size_t message_size() const; // Basic usage of the unpacker is as following: @@ -112,6 +112,7 @@ public: public: // These functions are usable when non-MessagePack message follows after // MessagePack message. + size_t parsed_size() const; /*! get address of the buffer that is not parsed */ char* nonparsed_buffer(); @@ -218,12 +219,17 @@ inline void unpacker::reset() msgpack_unpacker_reset(this); } +inline size_t unpacker::message_size() const +{ + return msgpack_unpacker_message_size(this); +} + + inline size_t unpacker::parsed_size() const { return msgpack_unpacker_parsed_size(this); } - inline char* unpacker::nonparsed_buffer() { return buf + off; From 980bb9cdd08bd3e5a0ff6021a45a1d880691c013 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 26 Feb 2009 01:27:00 +0900 Subject: [PATCH 0074/1172] add msgpack_unpacker_message_size --- c/unpack.h | 10 +++++++++- cpp/unpack.hpp | 12 +++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/c/unpack.h b/c/unpack.h index 77fbd54..779e256 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -61,7 +61,8 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); void msgpack_unpacker_reset(msgpack_unpacker* mpac); -static inline size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_message_size(const msgpack_unpacker* mpac); + typedef enum { @@ -76,6 +77,8 @@ msgpack_unpack(const char* data, size_t len, size_t* off, msgpack_zone* z, msgpack_object* result); +static inline size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac); + bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); @@ -102,6 +105,11 @@ void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size) mpac->free -= size; } +size_t msgpack_unpacker_message_size(const msgpack_unpacker* mpac) +{ + return mpac->parsed - mpac->off + mpac->used; +} + size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac) { return mpac->parsed; diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 0b7fe57..e140b36 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -67,8 +67,8 @@ public: /*! 5.3. after release_zone(), re-initialize unpacker */ void reset(); - /*! 6. check if the parsed message size doesn't exceed assumption. */ - size_t parsed_size() const; + /*! 6. check if the size of message doesn't exceed assumption. */ + size_t message_size() const; // Basic usage of the unpacker is as following: @@ -112,6 +112,7 @@ public: public: // These functions are usable when non-MessagePack message follows after // MessagePack message. + size_t parsed_size() const; /*! get address of the buffer that is not parsed */ char* nonparsed_buffer(); @@ -218,12 +219,17 @@ inline void unpacker::reset() msgpack_unpacker_reset(this); } +inline size_t unpacker::message_size() const +{ + return msgpack_unpacker_message_size(this); +} + + inline size_t unpacker::parsed_size() const { return msgpack_unpacker_parsed_size(this); } - inline char* unpacker::nonparsed_buffer() { return buf + off; From 9b4b49a6a85f9044ebaffff71b2498a3cf6c8ae4 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Thu, 26 Feb 2009 01:33:34 +0900 Subject: [PATCH 0075/1172] example: limit message size --- example/stream.cc | 4 ++++ example/stream.rb | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/example/stream.cc b/example/stream.cc index 8fd92f9..1e2e733 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -43,6 +43,10 @@ public: process_message(msg, life); } + + if(m_pac.message_size() > 10*1024*1024) { + throw std::runtime_error("message is too large"); + } } private: diff --git a/example/stream.rb b/example/stream.rb index e53ce82..a72f5b9 100644 --- a/example/stream.rb +++ b/example/stream.rb @@ -34,12 +34,17 @@ class Server @pk.reset @buffer.slice!(0, @nread) @nread = 0 + next unless @buffer.empty? end break end + if @buffer.length > 10*1024*1024 + raise "message is too large" + end + rescue puts "error while processing client packet: #{$!}" end From e7fe3be34dbae518af91eb671e4fee79bc32fb4e Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 26 Feb 2009 01:33:34 +0900 Subject: [PATCH 0076/1172] example: limit message size --- example/stream.cc | 4 ++++ example/stream.rb | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/example/stream.cc b/example/stream.cc index 8fd92f9..1e2e733 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -43,6 +43,10 @@ public: process_message(msg, life); } + + if(m_pac.message_size() > 10*1024*1024) { + throw std::runtime_error("message is too large"); + } } private: diff --git a/example/stream.rb b/example/stream.rb index e53ce82..a72f5b9 100644 --- a/example/stream.rb +++ b/example/stream.rb @@ -34,12 +34,17 @@ class Server @pk.reset @buffer.slice!(0, @nread) @nread = 0 + next unless @buffer.empty? end break end + if @buffer.length > 10*1024*1024 + raise "message is too large" + end + rescue puts "error while processing client packet: #{$!}" end From c60b6be5488df5b8218c1bbd0d8c3711d8dc73aa Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Thu, 26 Feb 2009 18:37:13 +0900 Subject: [PATCH 0077/1172] add NOTICE file --- LICENSE | 202 +++++++++++++++++++++++++++++++++++++++++++++++++ NOTICE | 4 + README | 38 +++++----- cpp/object.hpp | 2 - 4 files changed, 226 insertions(+), 20 deletions(-) create mode 100644 LICENSE create mode 100644 NOTICE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..e706f2a --- /dev/null +++ b/NOTICE @@ -0,0 +1,4 @@ +MessagePack is developed by FURUHASHI Sadayuki, licensed under Apache License, +Version 2.0. The original software and related information is available at +http://msgpack.sourceforge.jp/. + diff --git a/README b/README index 6e6ace4..0c1d440 100644 --- a/README +++ b/README @@ -17,41 +17,43 @@ Binary-based efficient data interchange format. Simply run ./configure && make && make install to install C and C++ binding. - $ ./configure --prefix=path/to/prefix - $ make - $ make install + $ ./configure + $ make + $ sudo make install - To install Ruby binding, run ./gengem.sh script in ruby/ directory and install + To install Ruby binding, run ./gengem.sh script on ruby/ directory and install generated gem package. - $ cd ruby - $ ./gengem.sh - $ gem install gem/pkg/msgpack-*.gem + $ cd ruby + $ ./gengem.sh + $ gem install gem/pkg/msgpack-*.gem *Usage C++: - #include - // TODO + include msgpack.hpp header and link libmsgpack library. + see example/simple.cc for example. + + g++ simple.cc -lmsgpack + g++ stream.cc -lmsgpack -lpthread C: - #include - /* TODO */ + include msgpack.h header and link libmsgpackc library. + see example/simple.c for example. + + gcc simple.c -lmsgpackc Ruby: - require 'msgpack' + require msgpack library. + see example/simple.rb for example. - # serialize - buf = [1, 2, 3].to_msgpack + ruby -rubygems simple.rb - # deserialize - p MessagePack::unpack(buf) - - # TODO + API Document is available at http://msgpack.sourceforge.jp/. Copyright (C) 2008-2009 FURUHASHI Sadayuki diff --git a/cpp/object.hpp b/cpp/object.hpp index b4c21f1..860b569 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -314,7 +314,6 @@ packer& operator<< (packer& o, const object& v) o << *p; } return o; - // FIXME loop optimiziation case type::MAP: o.pack_map(v.via.map.size); @@ -325,7 +324,6 @@ packer& operator<< (packer& o, const object& v) o << p->val; } return o; - // FIXME loop optimiziation default: throw type_error(); From 00dcad17b915129f3e4f91f885cd946451fd8e31 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 26 Feb 2009 18:37:13 +0900 Subject: [PATCH 0078/1172] add NOTICE file --- LICENSE | 202 +++++++++++++++++++++++++++++++++++++++++++++++++ NOTICE | 4 + README | 38 +++++----- cpp/object.hpp | 2 - 4 files changed, 226 insertions(+), 20 deletions(-) create mode 100644 LICENSE create mode 100644 NOTICE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..e706f2a --- /dev/null +++ b/NOTICE @@ -0,0 +1,4 @@ +MessagePack is developed by FURUHASHI Sadayuki, licensed under Apache License, +Version 2.0. The original software and related information is available at +http://msgpack.sourceforge.jp/. + diff --git a/README b/README index 6e6ace4..0c1d440 100644 --- a/README +++ b/README @@ -17,41 +17,43 @@ Binary-based efficient data interchange format. Simply run ./configure && make && make install to install C and C++ binding. - $ ./configure --prefix=path/to/prefix - $ make - $ make install + $ ./configure + $ make + $ sudo make install - To install Ruby binding, run ./gengem.sh script in ruby/ directory and install + To install Ruby binding, run ./gengem.sh script on ruby/ directory and install generated gem package. - $ cd ruby - $ ./gengem.sh - $ gem install gem/pkg/msgpack-*.gem + $ cd ruby + $ ./gengem.sh + $ gem install gem/pkg/msgpack-*.gem *Usage C++: - #include - // TODO + include msgpack.hpp header and link libmsgpack library. + see example/simple.cc for example. + + g++ simple.cc -lmsgpack + g++ stream.cc -lmsgpack -lpthread C: - #include - /* TODO */ + include msgpack.h header and link libmsgpackc library. + see example/simple.c for example. + + gcc simple.c -lmsgpackc Ruby: - require 'msgpack' + require msgpack library. + see example/simple.rb for example. - # serialize - buf = [1, 2, 3].to_msgpack + ruby -rubygems simple.rb - # deserialize - p MessagePack::unpack(buf) - - # TODO + API Document is available at http://msgpack.sourceforge.jp/. Copyright (C) 2008-2009 FURUHASHI Sadayuki diff --git a/cpp/object.hpp b/cpp/object.hpp index b4c21f1..860b569 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -314,7 +314,6 @@ packer& operator<< (packer& o, const object& v) o << *p; } return o; - // FIXME loop optimiziation case type::MAP: o.pack_map(v.via.map.size); @@ -325,7 +324,6 @@ packer& operator<< (packer& o, const object& v) o << p->val; } return o; - // FIXME loop optimiziation default: throw type_error(); From 4f4fa39cd569f4f4c329c55f1d84d33c53ee8dd4 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 00:59:15 +0900 Subject: [PATCH 0079/1172] c: msgpack_sbuffer; cpp: msgpack::sbuffer --- c/sbuffer.h | 37 ++++++++++++++++++------- cpp/sbuffer.hpp | 72 ++++++++++++++++++++++++++++++------------------ cpp/zone.hpp.erb | 1 - example/simple.c | 2 +- 4 files changed, 73 insertions(+), 39 deletions(-) diff --git a/c/sbuffer.h b/c/sbuffer.h index 3694dbd..8dae103 100644 --- a/c/sbuffer.h +++ b/c/sbuffer.h @@ -21,14 +21,18 @@ #include #include +#ifndef MSGPACK_SBUFFER_INIT_SIZE +#define MSGPACK_SBUFFER_INIT_SIZE 2048 +#endif + #ifdef __cplusplus extern "C" { #endif -typedef struct { - char* ptr; +typedef struct msgpack_sbuffer { size_t size; - size_t capacity; + char* data; + size_t alloc; } msgpack_sbuffer; static inline void msgpack_sbuffer_init(msgpack_sbuffer* sbuf) @@ -38,27 +42,40 @@ static inline void msgpack_sbuffer_init(msgpack_sbuffer* sbuf) static inline void msgpack_sbuffer_destroy(msgpack_sbuffer* sbuf) { - free(sbuf->ptr); + free(sbuf->data); } static inline int msgpack_sbuffer_write(void* data, const char* buf, unsigned int len) { msgpack_sbuffer* sbuf = (msgpack_sbuffer*)data; - if(sbuf->capacity - sbuf->size < len) { - size_t nsize = (sbuf->capacity ? sbuf->capacity*2 : 2048); + + if(sbuf->alloc - sbuf->size < len) { + size_t nsize = (sbuf->alloc) ? + sbuf->alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; + while(nsize < sbuf->size + len) { nsize *= 2; } - void* tmp = realloc(sbuf->ptr, nsize); + void* tmp = realloc(sbuf->data, nsize); if(!tmp) { return -1; } - sbuf->ptr = (char*)tmp; - sbuf->capacity = nsize; + sbuf->data = (char*)tmp; + sbuf->alloc = nsize; } - memcpy(sbuf->ptr + sbuf->size, buf, len); + + memcpy(sbuf->data + sbuf->size, buf, len); sbuf->size += len; return 0; } +static inline char* msgpack_sbuffer_release(msgpack_sbuffer* sbuf) +{ + char* tmp = sbuf->data; + sbuf->data = NULL; + sbuf->size = 0; + sbuf->alloc = NULL; + return tmp; +} + #ifdef __cplusplus } diff --git a/cpp/sbuffer.hpp b/cpp/sbuffer.hpp index 1dac6da..37ede49 100644 --- a/cpp/sbuffer.hpp +++ b/cpp/sbuffer.hpp @@ -18,62 +18,80 @@ #ifndef MSGPACK_SBUFFER_HPP__ #define MSGPACK_SBUFFER_HPP__ -#include -#include +#include "msgpack/sbuffer.h" #include namespace msgpack { -class sbuffer { +class sbuffer : public msgpack_sbuffer { public: - sbuffer() : m_capacity(0), m_size(0), m_ptr(NULL) { } + sbuffer(size_t initsz = MSGPACK_SBUFFER_INIT_SIZE) + { + msgpack_sbuffer* sbuf = static_cast(this); + + sbuf->data = (char*)::malloc(initsz); + if(!sbuf->data) { + throw std::bad_alloc(); + } + + sbuf->size = 0; + sbuf->alloc = initsz; + } ~sbuffer() { - free(m_ptr); + msgpack_sbuffer* sbuf = static_cast(this); + ::free(sbuf->data); } public: - void write(const char* buf, size_t len) + void write(const char* buf, unsigned int len) { - if(m_capacity - m_size < len) { - size_t nsize = (m_capacity ? m_capacity*2 : 2048); - while(nsize < m_size + len) { nsize *= 2; } - - char* tmp = (char*)realloc(m_ptr, nsize); - if(!tmp) { throw std::bad_alloc(); } - - m_ptr = tmp; - m_capacity = nsize; + msgpack_sbuffer* sbuf = static_cast(this); + if(sbuf->alloc - sbuf->size < len) { + expand_buffer(len); } - memcpy(m_ptr + m_size, buf, len); - m_size += len; + memcpy(sbuf->data + sbuf->size, buf, len); + sbuf->size += len; } char* data() { - return m_ptr; + msgpack_sbuffer* sbuf = static_cast(this); + return sbuf->data; } size_t size() const { - return m_size; + const msgpack_sbuffer* sbuf = static_cast(this); + return sbuf->size; } char* release() { - char* tmp = m_ptr; - m_capacity = 0; - m_size = 0; - m_ptr = NULL; - return tmp; + msgpack_sbuffer* sbuf = static_cast(this); + return msgpack_sbuffer_release(sbuf); } private: - size_t m_capacity; - size_t m_size; - char* m_ptr; + void expand_buffer(size_t len) + { + msgpack_sbuffer* sbuf = static_cast(this); + + size_t nsize = (sbuf->alloc) ? + sbuf->alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; + + while(nsize < sbuf->size + len) { nsize *= 2; } + + void* tmp = realloc(sbuf->data, nsize); + if(!tmp) { + throw std::bad_alloc(); + } + + sbuf->data = (char*)tmp; + sbuf->alloc = nsize; + } private: sbuffer(const sbuffer&); diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 06cb9d3..a253627 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -99,4 +99,3 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) #endif /* msgpack/zone.hpp */ -// vim: ft=cpp ts=4 sw=4 softtabstop=4 noexpandtab smarttab diff --git a/example/simple.c b/example/simple.c index df60424..41d8bb7 100644 --- a/example/simple.c +++ b/example/simple.c @@ -23,7 +23,7 @@ int main(void) msgpack_zone_init(&mempool, 2048); msgpack_object deserialized; - msgpack_unpack(sbuf.ptr, sbuf.size, NULL, &mempool, &deserialized); + msgpack_unpack(sbuf.data, sbuf.size, NULL, &mempool, &deserialized); /* print the deserialized object. */ msgpack_object_print(stdout, deserialized); From 76f18a0ea679af7f964ec455d93acc9b2c25b942 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 00:59:15 +0900 Subject: [PATCH 0080/1172] c: msgpack_sbuffer; cpp: msgpack::sbuffer --- c/sbuffer.h | 37 ++++++++++++++++++------- cpp/sbuffer.hpp | 72 ++++++++++++++++++++++++++++++------------------ cpp/zone.hpp.erb | 1 - example/simple.c | 2 +- 4 files changed, 73 insertions(+), 39 deletions(-) diff --git a/c/sbuffer.h b/c/sbuffer.h index 3694dbd..8dae103 100644 --- a/c/sbuffer.h +++ b/c/sbuffer.h @@ -21,14 +21,18 @@ #include #include +#ifndef MSGPACK_SBUFFER_INIT_SIZE +#define MSGPACK_SBUFFER_INIT_SIZE 2048 +#endif + #ifdef __cplusplus extern "C" { #endif -typedef struct { - char* ptr; +typedef struct msgpack_sbuffer { size_t size; - size_t capacity; + char* data; + size_t alloc; } msgpack_sbuffer; static inline void msgpack_sbuffer_init(msgpack_sbuffer* sbuf) @@ -38,27 +42,40 @@ static inline void msgpack_sbuffer_init(msgpack_sbuffer* sbuf) static inline void msgpack_sbuffer_destroy(msgpack_sbuffer* sbuf) { - free(sbuf->ptr); + free(sbuf->data); } static inline int msgpack_sbuffer_write(void* data, const char* buf, unsigned int len) { msgpack_sbuffer* sbuf = (msgpack_sbuffer*)data; - if(sbuf->capacity - sbuf->size < len) { - size_t nsize = (sbuf->capacity ? sbuf->capacity*2 : 2048); + + if(sbuf->alloc - sbuf->size < len) { + size_t nsize = (sbuf->alloc) ? + sbuf->alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; + while(nsize < sbuf->size + len) { nsize *= 2; } - void* tmp = realloc(sbuf->ptr, nsize); + void* tmp = realloc(sbuf->data, nsize); if(!tmp) { return -1; } - sbuf->ptr = (char*)tmp; - sbuf->capacity = nsize; + sbuf->data = (char*)tmp; + sbuf->alloc = nsize; } - memcpy(sbuf->ptr + sbuf->size, buf, len); + + memcpy(sbuf->data + sbuf->size, buf, len); sbuf->size += len; return 0; } +static inline char* msgpack_sbuffer_release(msgpack_sbuffer* sbuf) +{ + char* tmp = sbuf->data; + sbuf->data = NULL; + sbuf->size = 0; + sbuf->alloc = NULL; + return tmp; +} + #ifdef __cplusplus } diff --git a/cpp/sbuffer.hpp b/cpp/sbuffer.hpp index 1dac6da..37ede49 100644 --- a/cpp/sbuffer.hpp +++ b/cpp/sbuffer.hpp @@ -18,62 +18,80 @@ #ifndef MSGPACK_SBUFFER_HPP__ #define MSGPACK_SBUFFER_HPP__ -#include -#include +#include "msgpack/sbuffer.h" #include namespace msgpack { -class sbuffer { +class sbuffer : public msgpack_sbuffer { public: - sbuffer() : m_capacity(0), m_size(0), m_ptr(NULL) { } + sbuffer(size_t initsz = MSGPACK_SBUFFER_INIT_SIZE) + { + msgpack_sbuffer* sbuf = static_cast(this); + + sbuf->data = (char*)::malloc(initsz); + if(!sbuf->data) { + throw std::bad_alloc(); + } + + sbuf->size = 0; + sbuf->alloc = initsz; + } ~sbuffer() { - free(m_ptr); + msgpack_sbuffer* sbuf = static_cast(this); + ::free(sbuf->data); } public: - void write(const char* buf, size_t len) + void write(const char* buf, unsigned int len) { - if(m_capacity - m_size < len) { - size_t nsize = (m_capacity ? m_capacity*2 : 2048); - while(nsize < m_size + len) { nsize *= 2; } - - char* tmp = (char*)realloc(m_ptr, nsize); - if(!tmp) { throw std::bad_alloc(); } - - m_ptr = tmp; - m_capacity = nsize; + msgpack_sbuffer* sbuf = static_cast(this); + if(sbuf->alloc - sbuf->size < len) { + expand_buffer(len); } - memcpy(m_ptr + m_size, buf, len); - m_size += len; + memcpy(sbuf->data + sbuf->size, buf, len); + sbuf->size += len; } char* data() { - return m_ptr; + msgpack_sbuffer* sbuf = static_cast(this); + return sbuf->data; } size_t size() const { - return m_size; + const msgpack_sbuffer* sbuf = static_cast(this); + return sbuf->size; } char* release() { - char* tmp = m_ptr; - m_capacity = 0; - m_size = 0; - m_ptr = NULL; - return tmp; + msgpack_sbuffer* sbuf = static_cast(this); + return msgpack_sbuffer_release(sbuf); } private: - size_t m_capacity; - size_t m_size; - char* m_ptr; + void expand_buffer(size_t len) + { + msgpack_sbuffer* sbuf = static_cast(this); + + size_t nsize = (sbuf->alloc) ? + sbuf->alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; + + while(nsize < sbuf->size + len) { nsize *= 2; } + + void* tmp = realloc(sbuf->data, nsize); + if(!tmp) { + throw std::bad_alloc(); + } + + sbuf->data = (char*)tmp; + sbuf->alloc = nsize; + } private: sbuffer(const sbuffer&); diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 06cb9d3..a253627 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -99,4 +99,3 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) #endif /* msgpack/zone.hpp */ -// vim: ft=cpp ts=4 sw=4 softtabstop=4 noexpandtab smarttab diff --git a/example/simple.c b/example/simple.c index df60424..41d8bb7 100644 --- a/example/simple.c +++ b/example/simple.c @@ -23,7 +23,7 @@ int main(void) msgpack_zone_init(&mempool, 2048); msgpack_object deserialized; - msgpack_unpack(sbuf.ptr, sbuf.size, NULL, &mempool, &deserialized); + msgpack_unpack(sbuf.data, sbuf.size, NULL, &mempool, &deserialized); /* print the deserialized object. */ msgpack_object_print(stdout, deserialized); From 6fda01111e45bd17aa46a8e1ae862eb4b298c8da Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 01:06:16 +0900 Subject: [PATCH 0081/1172] msgpack::sbuffer::data() const --- cpp/sbuffer.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/sbuffer.hpp b/cpp/sbuffer.hpp index 37ede49..2651b58 100644 --- a/cpp/sbuffer.hpp +++ b/cpp/sbuffer.hpp @@ -62,6 +62,12 @@ public: return sbuf->data; } + const char* data() const + { + const msgpack_sbuffer* sbuf = static_cast(this); + return sbuf->data; + } + size_t size() const { const msgpack_sbuffer* sbuf = static_cast(this); From 17c2fa5a4ed436e9b9c5b6d186eea40875b68345 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 01:06:16 +0900 Subject: [PATCH 0082/1172] msgpack::sbuffer::data() const --- cpp/sbuffer.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/sbuffer.hpp b/cpp/sbuffer.hpp index 37ede49..2651b58 100644 --- a/cpp/sbuffer.hpp +++ b/cpp/sbuffer.hpp @@ -62,6 +62,12 @@ public: return sbuf->data; } + const char* data() const + { + const msgpack_sbuffer* sbuf = static_cast(this); + return sbuf->data; + } + size_t size() const { const msgpack_sbuffer* sbuf = static_cast(this); From 879c70f93a1d624b8f3f1646a593eeaf75a33613 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 01:13:31 +0900 Subject: [PATCH 0083/1172] fix msgpack_sbuffer_release --- c/sbuffer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c/sbuffer.h b/c/sbuffer.h index 8dae103..b0fab46 100644 --- a/c/sbuffer.h +++ b/c/sbuffer.h @@ -70,9 +70,9 @@ static inline int msgpack_sbuffer_write(void* data, const char* buf, unsigned in static inline char* msgpack_sbuffer_release(msgpack_sbuffer* sbuf) { char* tmp = sbuf->data; - sbuf->data = NULL; sbuf->size = 0; - sbuf->alloc = NULL; + sbuf->data = NULL; + sbuf->alloc = 0; return tmp; } From 4f2755366faef06ebf9bff17ea15e2a53ec15649 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 01:13:31 +0900 Subject: [PATCH 0084/1172] fix msgpack_sbuffer_release --- c/sbuffer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c/sbuffer.h b/c/sbuffer.h index 8dae103..b0fab46 100644 --- a/c/sbuffer.h +++ b/c/sbuffer.h @@ -70,9 +70,9 @@ static inline int msgpack_sbuffer_write(void* data, const char* buf, unsigned in static inline char* msgpack_sbuffer_release(msgpack_sbuffer* sbuf) { char* tmp = sbuf->data; - sbuf->data = NULL; sbuf->size = 0; - sbuf->alloc = NULL; + sbuf->data = NULL; + sbuf->alloc = 0; return tmp; } From 6fc38d1669a9bdd5f14c0d6a261ce0b6c04d0ecc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 01:31:12 +0900 Subject: [PATCH 0085/1172] msgpack::pack is not obsolete --- cpp/object.hpp | 10 ---------- cpp/pack.hpp | 7 +++++++ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 860b569..09ddb89 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -110,9 +110,6 @@ bool operator!=(const object x, const object y); std::ostream& operator<< (std::ostream& s, const object o); -template -inline void pack(Stream& s, const T& v); - template packer& operator<< (packer& o, const T& v); @@ -238,13 +235,6 @@ inline void pack(packer& o, const T& v) o.pack(v); } -// obsolete -template -inline void pack(Stream& s, const T& v) -{ - packer(s).pack(v); -} - // obsolete template inline void pack_copy(packer& o, T v) diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 257ccb6..c8e37eb 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -112,6 +112,13 @@ private: }; +template +inline void pack(Stream& s, const T& v) +{ + packer(s).pack(v); +} + + #define msgpack_pack_inline_func(name) \ template \ inline void packer::_pack ## name From a8545b49c9da6ca75374973751e519d4da0bbf20 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 01:31:12 +0900 Subject: [PATCH 0086/1172] msgpack::pack is not obsolete --- cpp/object.hpp | 10 ---------- cpp/pack.hpp | 7 +++++++ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 860b569..09ddb89 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -110,9 +110,6 @@ bool operator!=(const object x, const object y); std::ostream& operator<< (std::ostream& s, const object o); -template -inline void pack(Stream& s, const T& v); - template packer& operator<< (packer& o, const T& v); @@ -238,13 +235,6 @@ inline void pack(packer& o, const T& v) o.pack(v); } -// obsolete -template -inline void pack(Stream& s, const T& v) -{ - packer(s).pack(v); -} - // obsolete template inline void pack_copy(packer& o, T v) diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 257ccb6..c8e37eb 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -112,6 +112,13 @@ private: }; +template +inline void pack(Stream& s, const T& v) +{ + packer(s).pack(v); +} + + #define msgpack_pack_inline_func(name) \ template \ inline void packer::_pack ## name From bf13ba72b599e78ae4aa70f88fee7346e6d76dac Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 02:27:04 +0900 Subject: [PATCH 0087/1172] fix unpacker --- c/unpack.c | 84 ++++++++++++++++++++++++------------------------- c/unpack.h | 5 ++- cpp/sbuffer.hpp | 49 +++++++++++++---------------- cpp/unpack.hpp | 19 ++++++----- 4 files changed, 76 insertions(+), 81 deletions(-) diff --git a/c/unpack.c b/c/unpack.c index 865e0d4..e90a29a 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -22,7 +22,7 @@ typedef struct { msgpack_zone* z; - bool* referenced; + bool referenced; } unpack_user; @@ -131,7 +131,7 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha o->type = MSGPACK_OBJECT_RAW; o->via.raw.ptr = p; o->via.raw.size = l; - *u->referenced = true; + u->referenced = true; return 0; } @@ -139,32 +139,33 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha #define CTX_CAST(m) ((template_context*)(m)) +#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced static const size_t COUNTER_SIZE = sizeof(unsigned int); -static inline void init_count(void* buf) +static inline void init_count(void* buffer) { - *(volatile unsigned int*)buf = 1; + *(volatile unsigned int*)buffer = 1; } -static inline void decl_count(void* buf) +static inline void decl_count(void* buffer) { - //if(--*(unsigned int*)buf == 0) { - if(__sync_sub_and_fetch((unsigned int*)buf, 1) == 0) { - free(buf); + //if(--*(unsigned int*)buffer == 0) { + if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { + free(buffer); } } -static inline void incr_count(void* buf) +static inline void incr_count(void* buffer) { - //++*(unsigned int*)buf; - __sync_add_and_fetch((unsigned int*)buf, 1); + //++*(unsigned int*)buffer; + __sync_add_and_fetch((unsigned int*)buffer, 1); } -static inline unsigned int get_count(void* buf) +static inline unsigned int get_count(void* buffer) { - return *(volatile unsigned int*)buf; + return *(volatile unsigned int*)buffer; } @@ -175,39 +176,38 @@ bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) initial_buffer_size = COUNTER_SIZE; } - char* buf = (char*)malloc(initial_buffer_size); - if(buf == NULL) { + char* buffer = (char*)malloc(initial_buffer_size); + if(buffer == NULL) { return false; } void* ctx = malloc(sizeof(template_context)); if(ctx == NULL) { - free(buf); + free(buffer); return false; } msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); if(z == NULL) { free(ctx); - free(buf); + free(buffer); return false; } - mpac->buf = buf; + mpac->buffer = buffer; mpac->used = COUNTER_SIZE; mpac->free = initial_buffer_size - mpac->used; mpac->off = COUNTER_SIZE; mpac->parsed = 0; mpac->initial_buffer_size = initial_buffer_size; mpac->z = z; - mpac->referenced = false; mpac->ctx = ctx; - init_count(mpac->buf); + init_count(mpac->buffer); template_init(CTX_CAST(mpac->ctx)); CTX_CAST(mpac->ctx)->user.z = mpac->z; - CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; + CTX_CAST(mpac->ctx)->user.referenced = false; return true; } @@ -216,7 +216,7 @@ void msgpack_unpacker_destroy(msgpack_unpacker* mpac) { msgpack_zone_free(mpac->z); free(mpac->ctx); - decl_count(mpac->buf); + decl_count(mpac->buffer); } @@ -241,10 +241,10 @@ void msgpack_unpacker_free(msgpack_unpacker* mpac) free(mpac); } - bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) { - if(mpac->used == mpac->off && get_count(mpac->buf) == 1 && !mpac->referenced) { + if(mpac->used == mpac->off && get_count(mpac->buffer) == 1 + && !CTX_REFERENCED(mpac)) { // rewind buffer mpac->free += mpac->used - COUNTER_SIZE; mpac->used = COUNTER_SIZE; @@ -261,12 +261,12 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) next_size *= 2; } - char* tmp = (char*)realloc(mpac->buf, next_size); + char* tmp = (char*)realloc(mpac->buffer, next_size); if(tmp == NULL) { return false; } - mpac->buf = tmp; + mpac->buffer = tmp; mpac->free = next_size - mpac->used; } else { @@ -283,19 +283,19 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) init_count(tmp); - if(mpac->referenced) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { free(tmp); return false; } - mpac->referenced = false; + CTX_REFERENCED(mpac) = false; } else { - decl_count(mpac->buf); + decl_count(mpac->buffer); } - memcpy(tmp+COUNTER_SIZE, mpac->buf+mpac->off, not_parsed); + memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); - mpac->buf = tmp; + mpac->buffer = tmp; mpac->used = not_parsed + COUNTER_SIZE; mpac->free = next_size - mpac->used; mpac->off = COUNTER_SIZE; @@ -308,7 +308,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* mpac) { size_t off = mpac->off; int ret = template_execute(CTX_CAST(mpac->ctx), - mpac->buf, mpac->used, &mpac->off); + mpac->buffer, mpac->used, &mpac->off); if(mpac->off > off) { mpac->parsed += mpac->off - off; } @@ -326,26 +326,26 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) return false; } - msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); - if(z == NULL) { + msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(r == NULL) { return NULL; } msgpack_zone* old = mpac->z; - mpac->z = z; + mpac->z = r; return old; } bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) { - if(mpac->referenced) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { return false; } - mpac->referenced = false; + CTX_REFERENCED(mpac) = false; - incr_count(mpac->buf); + incr_count(mpac->buffer); } return true; @@ -354,6 +354,7 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) void msgpack_unpacker_reset(msgpack_unpacker* mpac) { template_init(CTX_CAST(mpac->ctx)); + // don't reset referenced flag mpac->parsed = 0; } @@ -365,9 +366,8 @@ msgpack_unpack(const char* data, size_t len, size_t* off, template_context ctx; template_init(&ctx); - bool referenced = false; ctx.user.z = z; - ctx.user.referenced = &referenced; + ctx.user.referenced = false; size_t noff = 0; if(off != NULL) { noff = *off; } diff --git a/c/unpack.h b/c/unpack.h index 779e256..a9caf07 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -29,13 +29,12 @@ extern "C" { typedef struct msgpack_unpacker { - char* buf; + char* buffer; size_t used; size_t free; size_t off; size_t parsed; msgpack_zone* z; - bool referenced; size_t initial_buffer_size; void* ctx; } msgpack_unpacker; @@ -91,7 +90,7 @@ bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) char* msgpack_unpacker_buffer(msgpack_unpacker* mpac) { - return mpac->buf + mpac->used; + return mpac->buffer + mpac->used; } size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac) diff --git a/cpp/sbuffer.hpp b/cpp/sbuffer.hpp index 2651b58..ca06884e 100644 --- a/cpp/sbuffer.hpp +++ b/cpp/sbuffer.hpp @@ -28,77 +28,70 @@ class sbuffer : public msgpack_sbuffer { public: sbuffer(size_t initsz = MSGPACK_SBUFFER_INIT_SIZE) { - msgpack_sbuffer* sbuf = static_cast(this); - - sbuf->data = (char*)::malloc(initsz); - if(!sbuf->data) { + base::data = (char*)::malloc(initsz); + if(!base::data) { throw std::bad_alloc(); } - sbuf->size = 0; - sbuf->alloc = initsz; + base::size = 0; + base::alloc = initsz; } ~sbuffer() { - msgpack_sbuffer* sbuf = static_cast(this); - ::free(sbuf->data); + ::free(base::data); } public: void write(const char* buf, unsigned int len) { - msgpack_sbuffer* sbuf = static_cast(this); - if(sbuf->alloc - sbuf->size < len) { + if(base::alloc - base::size < len) { expand_buffer(len); } - memcpy(sbuf->data + sbuf->size, buf, len); - sbuf->size += len; + memcpy(base::data + base::size, buf, len); + base::size += len; } char* data() { - msgpack_sbuffer* sbuf = static_cast(this); - return sbuf->data; + return base::data; } const char* data() const { - const msgpack_sbuffer* sbuf = static_cast(this); - return sbuf->data; + return base::data; } size_t size() const { - const msgpack_sbuffer* sbuf = static_cast(this); - return sbuf->size; + return base::size; } char* release() { - msgpack_sbuffer* sbuf = static_cast(this); - return msgpack_sbuffer_release(sbuf); + return msgpack_sbuffer_release(this); } private: void expand_buffer(size_t len) { - msgpack_sbuffer* sbuf = static_cast(this); - - size_t nsize = (sbuf->alloc) ? - sbuf->alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; + size_t nsize = (base::alloc) ? + base::alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; - while(nsize < sbuf->size + len) { nsize *= 2; } + while(nsize < base::size + len) { nsize *= 2; } - void* tmp = realloc(sbuf->data, nsize); + void* tmp = realloc(base::data, nsize); if(!tmp) { throw std::bad_alloc(); } - sbuf->data = (char*)tmp; - sbuf->alloc = nsize; + base::data = (char*)tmp; + base::alloc = nsize; } +private: + typedef msgpack_sbuffer base; + private: sbuffer(const sbuffer&); }; diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index e140b36..38ac7ac 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -39,7 +39,7 @@ struct unpack_error : public std::runtime_error { class unpacker : public msgpack_unpacker { public: - unpacker(size_t initial_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE); + unpacker(size_t init_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE); ~unpacker(); public: @@ -126,6 +126,9 @@ public: // Note that reset() leaves non-parsed buffer. void remove_nonparsed_buffer(); +private: + typedef msgpack_unpacker base; + private: unpacker(const unpacker&); }; @@ -207,9 +210,9 @@ inline zone* unpacker::release_zone() zone* r = new zone(); - msgpack_zone old = *this->z; - *this->z = *z; - *z = old; + msgpack_zone old = *base::z; + *base::z = *r; + *static_cast(r) = old; return r; } @@ -232,22 +235,22 @@ inline size_t unpacker::parsed_size() const inline char* unpacker::nonparsed_buffer() { - return buf + off; + return base::buffer + base::off; } inline size_t unpacker::nonparsed_size() const { - return used - off; + return base::used - base::off; } inline void unpacker::skip_nonparsed_buffer(size_t size) { - off += size; + base::off += size; } inline void unpacker::remove_nonparsed_buffer() { - used = off; + base::used = base::off; } From ef1c4f82b27e2690694e1995828ca374b5cd225e Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 02:27:04 +0900 Subject: [PATCH 0088/1172] fix unpacker --- c/unpack.c | 84 ++++++++++++++++++++++++------------------------- c/unpack.h | 5 ++- cpp/sbuffer.hpp | 49 +++++++++++++---------------- cpp/unpack.hpp | 19 ++++++----- 4 files changed, 76 insertions(+), 81 deletions(-) diff --git a/c/unpack.c b/c/unpack.c index 865e0d4..e90a29a 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -22,7 +22,7 @@ typedef struct { msgpack_zone* z; - bool* referenced; + bool referenced; } unpack_user; @@ -131,7 +131,7 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha o->type = MSGPACK_OBJECT_RAW; o->via.raw.ptr = p; o->via.raw.size = l; - *u->referenced = true; + u->referenced = true; return 0; } @@ -139,32 +139,33 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha #define CTX_CAST(m) ((template_context*)(m)) +#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced static const size_t COUNTER_SIZE = sizeof(unsigned int); -static inline void init_count(void* buf) +static inline void init_count(void* buffer) { - *(volatile unsigned int*)buf = 1; + *(volatile unsigned int*)buffer = 1; } -static inline void decl_count(void* buf) +static inline void decl_count(void* buffer) { - //if(--*(unsigned int*)buf == 0) { - if(__sync_sub_and_fetch((unsigned int*)buf, 1) == 0) { - free(buf); + //if(--*(unsigned int*)buffer == 0) { + if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { + free(buffer); } } -static inline void incr_count(void* buf) +static inline void incr_count(void* buffer) { - //++*(unsigned int*)buf; - __sync_add_and_fetch((unsigned int*)buf, 1); + //++*(unsigned int*)buffer; + __sync_add_and_fetch((unsigned int*)buffer, 1); } -static inline unsigned int get_count(void* buf) +static inline unsigned int get_count(void* buffer) { - return *(volatile unsigned int*)buf; + return *(volatile unsigned int*)buffer; } @@ -175,39 +176,38 @@ bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) initial_buffer_size = COUNTER_SIZE; } - char* buf = (char*)malloc(initial_buffer_size); - if(buf == NULL) { + char* buffer = (char*)malloc(initial_buffer_size); + if(buffer == NULL) { return false; } void* ctx = malloc(sizeof(template_context)); if(ctx == NULL) { - free(buf); + free(buffer); return false; } msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); if(z == NULL) { free(ctx); - free(buf); + free(buffer); return false; } - mpac->buf = buf; + mpac->buffer = buffer; mpac->used = COUNTER_SIZE; mpac->free = initial_buffer_size - mpac->used; mpac->off = COUNTER_SIZE; mpac->parsed = 0; mpac->initial_buffer_size = initial_buffer_size; mpac->z = z; - mpac->referenced = false; mpac->ctx = ctx; - init_count(mpac->buf); + init_count(mpac->buffer); template_init(CTX_CAST(mpac->ctx)); CTX_CAST(mpac->ctx)->user.z = mpac->z; - CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; + CTX_CAST(mpac->ctx)->user.referenced = false; return true; } @@ -216,7 +216,7 @@ void msgpack_unpacker_destroy(msgpack_unpacker* mpac) { msgpack_zone_free(mpac->z); free(mpac->ctx); - decl_count(mpac->buf); + decl_count(mpac->buffer); } @@ -241,10 +241,10 @@ void msgpack_unpacker_free(msgpack_unpacker* mpac) free(mpac); } - bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) { - if(mpac->used == mpac->off && get_count(mpac->buf) == 1 && !mpac->referenced) { + if(mpac->used == mpac->off && get_count(mpac->buffer) == 1 + && !CTX_REFERENCED(mpac)) { // rewind buffer mpac->free += mpac->used - COUNTER_SIZE; mpac->used = COUNTER_SIZE; @@ -261,12 +261,12 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) next_size *= 2; } - char* tmp = (char*)realloc(mpac->buf, next_size); + char* tmp = (char*)realloc(mpac->buffer, next_size); if(tmp == NULL) { return false; } - mpac->buf = tmp; + mpac->buffer = tmp; mpac->free = next_size - mpac->used; } else { @@ -283,19 +283,19 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) init_count(tmp); - if(mpac->referenced) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { free(tmp); return false; } - mpac->referenced = false; + CTX_REFERENCED(mpac) = false; } else { - decl_count(mpac->buf); + decl_count(mpac->buffer); } - memcpy(tmp+COUNTER_SIZE, mpac->buf+mpac->off, not_parsed); + memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); - mpac->buf = tmp; + mpac->buffer = tmp; mpac->used = not_parsed + COUNTER_SIZE; mpac->free = next_size - mpac->used; mpac->off = COUNTER_SIZE; @@ -308,7 +308,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* mpac) { size_t off = mpac->off; int ret = template_execute(CTX_CAST(mpac->ctx), - mpac->buf, mpac->used, &mpac->off); + mpac->buffer, mpac->used, &mpac->off); if(mpac->off > off) { mpac->parsed += mpac->off - off; } @@ -326,26 +326,26 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) return false; } - msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); - if(z == NULL) { + msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(r == NULL) { return NULL; } msgpack_zone* old = mpac->z; - mpac->z = z; + mpac->z = r; return old; } bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) { - if(mpac->referenced) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { return false; } - mpac->referenced = false; + CTX_REFERENCED(mpac) = false; - incr_count(mpac->buf); + incr_count(mpac->buffer); } return true; @@ -354,6 +354,7 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) void msgpack_unpacker_reset(msgpack_unpacker* mpac) { template_init(CTX_CAST(mpac->ctx)); + // don't reset referenced flag mpac->parsed = 0; } @@ -365,9 +366,8 @@ msgpack_unpack(const char* data, size_t len, size_t* off, template_context ctx; template_init(&ctx); - bool referenced = false; ctx.user.z = z; - ctx.user.referenced = &referenced; + ctx.user.referenced = false; size_t noff = 0; if(off != NULL) { noff = *off; } diff --git a/c/unpack.h b/c/unpack.h index 779e256..a9caf07 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -29,13 +29,12 @@ extern "C" { typedef struct msgpack_unpacker { - char* buf; + char* buffer; size_t used; size_t free; size_t off; size_t parsed; msgpack_zone* z; - bool referenced; size_t initial_buffer_size; void* ctx; } msgpack_unpacker; @@ -91,7 +90,7 @@ bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) char* msgpack_unpacker_buffer(msgpack_unpacker* mpac) { - return mpac->buf + mpac->used; + return mpac->buffer + mpac->used; } size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac) diff --git a/cpp/sbuffer.hpp b/cpp/sbuffer.hpp index 2651b58..ca06884e 100644 --- a/cpp/sbuffer.hpp +++ b/cpp/sbuffer.hpp @@ -28,77 +28,70 @@ class sbuffer : public msgpack_sbuffer { public: sbuffer(size_t initsz = MSGPACK_SBUFFER_INIT_SIZE) { - msgpack_sbuffer* sbuf = static_cast(this); - - sbuf->data = (char*)::malloc(initsz); - if(!sbuf->data) { + base::data = (char*)::malloc(initsz); + if(!base::data) { throw std::bad_alloc(); } - sbuf->size = 0; - sbuf->alloc = initsz; + base::size = 0; + base::alloc = initsz; } ~sbuffer() { - msgpack_sbuffer* sbuf = static_cast(this); - ::free(sbuf->data); + ::free(base::data); } public: void write(const char* buf, unsigned int len) { - msgpack_sbuffer* sbuf = static_cast(this); - if(sbuf->alloc - sbuf->size < len) { + if(base::alloc - base::size < len) { expand_buffer(len); } - memcpy(sbuf->data + sbuf->size, buf, len); - sbuf->size += len; + memcpy(base::data + base::size, buf, len); + base::size += len; } char* data() { - msgpack_sbuffer* sbuf = static_cast(this); - return sbuf->data; + return base::data; } const char* data() const { - const msgpack_sbuffer* sbuf = static_cast(this); - return sbuf->data; + return base::data; } size_t size() const { - const msgpack_sbuffer* sbuf = static_cast(this); - return sbuf->size; + return base::size; } char* release() { - msgpack_sbuffer* sbuf = static_cast(this); - return msgpack_sbuffer_release(sbuf); + return msgpack_sbuffer_release(this); } private: void expand_buffer(size_t len) { - msgpack_sbuffer* sbuf = static_cast(this); - - size_t nsize = (sbuf->alloc) ? - sbuf->alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; + size_t nsize = (base::alloc) ? + base::alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; - while(nsize < sbuf->size + len) { nsize *= 2; } + while(nsize < base::size + len) { nsize *= 2; } - void* tmp = realloc(sbuf->data, nsize); + void* tmp = realloc(base::data, nsize); if(!tmp) { throw std::bad_alloc(); } - sbuf->data = (char*)tmp; - sbuf->alloc = nsize; + base::data = (char*)tmp; + base::alloc = nsize; } +private: + typedef msgpack_sbuffer base; + private: sbuffer(const sbuffer&); }; diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index e140b36..38ac7ac 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -39,7 +39,7 @@ struct unpack_error : public std::runtime_error { class unpacker : public msgpack_unpacker { public: - unpacker(size_t initial_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE); + unpacker(size_t init_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE); ~unpacker(); public: @@ -126,6 +126,9 @@ public: // Note that reset() leaves non-parsed buffer. void remove_nonparsed_buffer(); +private: + typedef msgpack_unpacker base; + private: unpacker(const unpacker&); }; @@ -207,9 +210,9 @@ inline zone* unpacker::release_zone() zone* r = new zone(); - msgpack_zone old = *this->z; - *this->z = *z; - *z = old; + msgpack_zone old = *base::z; + *base::z = *r; + *static_cast(r) = old; return r; } @@ -232,22 +235,22 @@ inline size_t unpacker::parsed_size() const inline char* unpacker::nonparsed_buffer() { - return buf + off; + return base::buffer + base::off; } inline size_t unpacker::nonparsed_size() const { - return used - off; + return base::used - base::off; } inline void unpacker::skip_nonparsed_buffer(size_t size) { - off += size; + base::off += size; } inline void unpacker::remove_nonparsed_buffer() { - used = off; + base::used = base::off; } From 0efb8160b97c78b67329436a45203e53e0f5cf2d Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 02:55:45 +0900 Subject: [PATCH 0089/1172] msgpack_zone_clear, msgpack::zone::clear --- c/unpack.c | 5 +++++ c/unpack.h | 2 ++ c/zone.c | 32 +++++++++++++++++++++++++++++++- c/zone.h | 2 ++ cpp/unpack.hpp | 8 ++++++++ cpp/zone.hpp.erb | 7 +++++++ 6 files changed, 55 insertions(+), 1 deletion(-) diff --git a/c/unpack.c b/c/unpack.c index e90a29a..f1b3bb1 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -337,6 +337,11 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) return old; } +void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac) +{ + msgpack_zone_clear(mpac->z); +} + bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) { if(CTX_REFERENCED(mpac)) { diff --git a/c/unpack.h b/c/unpack.h index a9caf07..ef63774 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -58,6 +58,8 @@ msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac); msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); +void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac); + void msgpack_unpacker_reset(msgpack_unpacker* mpac); static inline size_t msgpack_unpacker_message_size(const msgpack_unpacker* mpac); diff --git a/c/zone.c b/c/zone.c index e891c82..7a275f6 100644 --- a/c/zone.c +++ b/c/zone.c @@ -57,9 +57,23 @@ static inline void destroy_chunk_array(msgpack_zone_chunk_array* ca) for(; chunk != ca->tail+1; ++chunk) { free(chunk->alloc); } + free(ca->array); } +static inline void clear_chunk_array(msgpack_zone_chunk_array* ca) +{ + msgpack_zone_chunk* chunk = ca->array + 1; + for(; chunk != ca->tail+1; ++chunk) { + free(chunk->alloc); + } + + ca->tail = ca->array; + + ca->array[0].free += ca->array[0].ptr - (char*)ca->array[0].alloc; + ca->array[0].ptr = (char*)ca->array[0].alloc; +} + void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) { msgpack_zone_chunk_array* const ca = &zone->chunk_array; @@ -124,16 +138,27 @@ static inline void init_finalizer_array(msgpack_zone_finalizer_array* fa) fa->array = NULL; } -static inline void destroy_finalizer_array(msgpack_zone_finalizer_array* fa) +static inline void call_finalizer_array(msgpack_zone_finalizer_array* fa) { // 逆順ã«å‘¼ã³å‡ºã— msgpack_zone_finalizer* fin = fa->tail; for(; fin != fa->array; --fin) { (*(fin-1)->func)((fin-1)->data); } +} + +static inline void destroy_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + call_finalizer_array(fa); free(fa->array); } +static inline void clear_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + call_finalizer_array(fa); + fa->tail = fa->array; +} + bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data) { @@ -208,6 +233,11 @@ void msgpack_zone_destroy(msgpack_zone* zone) destroy_chunk_array(&zone->chunk_array); } +void msgpack_zone_clear(msgpack_zone* zone) +{ + clear_finalizer_array(&zone->finalizer_array); + clear_chunk_array(&zone->chunk_array); +} msgpack_zone* msgpack_zone_new(size_t chunk_size) { diff --git a/c/zone.h b/c/zone.h index 3dc9f52..3c1188d 100644 --- a/c/zone.h +++ b/c/zone.h @@ -72,6 +72,8 @@ bool msgpack_zone_push_finalizer(msgpack_zone* zone, bool msgpack_zone_is_empty(msgpack_zone* zone); +void msgpack_zone_clear(msgpack_zone* zone); + #ifdef __cplusplus } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 38ac7ac..324111a 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -64,6 +64,9 @@ public: // otherwise the memrory will leak. zone* release_zone(); + /*! 5.2. this method is equivalence to `delete release_zone()` */ + void reset_zone(); + /*! 5.3. after release_zone(), re-initialize unpacker */ void reset(); @@ -217,6 +220,11 @@ inline zone* unpacker::release_zone() return r; } +inline void unpacker::reset_zone() +{ + msgpack_unpacker_reset_zone(this); +} + inline void unpacker::reset() { msgpack_unpacker_reset(this); diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index a253627..930c8e8 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -37,6 +37,8 @@ public: void push_finalizer(void (*func)(void*), void* data); + void clear(); + <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> T* allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>); @@ -78,6 +80,11 @@ inline void zone::push_finalizer(void (*func)(void*), void* data) } } +inline void zone::clear() +{ + msgpack_zone_clear(this); +} + template void zone::object_destructor(void* obj) { From b33ecbd92baa866803e34e9d7f1148007b16f180 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 02:55:45 +0900 Subject: [PATCH 0090/1172] msgpack_zone_clear, msgpack::zone::clear --- c/unpack.c | 5 +++++ c/unpack.h | 2 ++ c/zone.c | 32 +++++++++++++++++++++++++++++++- c/zone.h | 2 ++ cpp/unpack.hpp | 8 ++++++++ cpp/zone.hpp.erb | 7 +++++++ 6 files changed, 55 insertions(+), 1 deletion(-) diff --git a/c/unpack.c b/c/unpack.c index e90a29a..f1b3bb1 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -337,6 +337,11 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) return old; } +void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac) +{ + msgpack_zone_clear(mpac->z); +} + bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) { if(CTX_REFERENCED(mpac)) { diff --git a/c/unpack.h b/c/unpack.h index a9caf07..ef63774 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -58,6 +58,8 @@ msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac); msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); +void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac); + void msgpack_unpacker_reset(msgpack_unpacker* mpac); static inline size_t msgpack_unpacker_message_size(const msgpack_unpacker* mpac); diff --git a/c/zone.c b/c/zone.c index e891c82..7a275f6 100644 --- a/c/zone.c +++ b/c/zone.c @@ -57,9 +57,23 @@ static inline void destroy_chunk_array(msgpack_zone_chunk_array* ca) for(; chunk != ca->tail+1; ++chunk) { free(chunk->alloc); } + free(ca->array); } +static inline void clear_chunk_array(msgpack_zone_chunk_array* ca) +{ + msgpack_zone_chunk* chunk = ca->array + 1; + for(; chunk != ca->tail+1; ++chunk) { + free(chunk->alloc); + } + + ca->tail = ca->array; + + ca->array[0].free += ca->array[0].ptr - (char*)ca->array[0].alloc; + ca->array[0].ptr = (char*)ca->array[0].alloc; +} + void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) { msgpack_zone_chunk_array* const ca = &zone->chunk_array; @@ -124,16 +138,27 @@ static inline void init_finalizer_array(msgpack_zone_finalizer_array* fa) fa->array = NULL; } -static inline void destroy_finalizer_array(msgpack_zone_finalizer_array* fa) +static inline void call_finalizer_array(msgpack_zone_finalizer_array* fa) { // 逆順ã«å‘¼ã³å‡ºã— msgpack_zone_finalizer* fin = fa->tail; for(; fin != fa->array; --fin) { (*(fin-1)->func)((fin-1)->data); } +} + +static inline void destroy_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + call_finalizer_array(fa); free(fa->array); } +static inline void clear_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + call_finalizer_array(fa); + fa->tail = fa->array; +} + bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data) { @@ -208,6 +233,11 @@ void msgpack_zone_destroy(msgpack_zone* zone) destroy_chunk_array(&zone->chunk_array); } +void msgpack_zone_clear(msgpack_zone* zone) +{ + clear_finalizer_array(&zone->finalizer_array); + clear_chunk_array(&zone->chunk_array); +} msgpack_zone* msgpack_zone_new(size_t chunk_size) { diff --git a/c/zone.h b/c/zone.h index 3dc9f52..3c1188d 100644 --- a/c/zone.h +++ b/c/zone.h @@ -72,6 +72,8 @@ bool msgpack_zone_push_finalizer(msgpack_zone* zone, bool msgpack_zone_is_empty(msgpack_zone* zone); +void msgpack_zone_clear(msgpack_zone* zone); + #ifdef __cplusplus } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 38ac7ac..324111a 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -64,6 +64,9 @@ public: // otherwise the memrory will leak. zone* release_zone(); + /*! 5.2. this method is equivalence to `delete release_zone()` */ + void reset_zone(); + /*! 5.3. after release_zone(), re-initialize unpacker */ void reset(); @@ -217,6 +220,11 @@ inline zone* unpacker::release_zone() return r; } +inline void unpacker::reset_zone() +{ + msgpack_unpacker_reset_zone(this); +} + inline void unpacker::reset() { msgpack_unpacker_reset(this); diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index a253627..930c8e8 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -37,6 +37,8 @@ public: void push_finalizer(void (*func)(void*), void* data); + void clear(); + <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> T* allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>); @@ -78,6 +80,11 @@ inline void zone::push_finalizer(void (*func)(void*), void* data) } } +inline void zone::clear() +{ + msgpack_zone_clear(this); +} + template void zone::object_destructor(void* obj) { From cb24a0c24f79adc6d3646cbd5b63005029e8fa0c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 03:39:34 +0900 Subject: [PATCH 0091/1172] fix msgpack_unpacker_expand_buffer --- c/unpack.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c/unpack.c b/c/unpack.c index f1b3bb1..08fd6cb 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -283,6 +283,8 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) init_count(tmp); + memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); + if(CTX_REFERENCED(mpac)) { if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { free(tmp); @@ -293,8 +295,6 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) decl_count(mpac->buffer); } - memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); - mpac->buffer = tmp; mpac->used = not_parsed + COUNTER_SIZE; mpac->free = next_size - mpac->used; From 87724d5b51b33968dc7db0bdc520c6405f50a5a7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 03:39:34 +0900 Subject: [PATCH 0092/1172] fix msgpack_unpacker_expand_buffer --- c/unpack.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c/unpack.c b/c/unpack.c index f1b3bb1..08fd6cb 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -283,6 +283,8 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) init_count(tmp); + memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); + if(CTX_REFERENCED(mpac)) { if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { free(tmp); @@ -293,8 +295,6 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) decl_count(mpac->buffer); } - memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); - mpac->buffer = tmp; mpac->used = not_parsed + COUNTER_SIZE; mpac->free = next_size - mpac->used; From c3f9696268e94bea44427c31c592402d54f02358 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 13:46:20 +0900 Subject: [PATCH 0093/1172] fix packaging problem --- Makefile.am | 17 +++++++++++++++++ cpp/Makefile.am | 10 +++++----- ruby/gem/AUTHORS | 1 - ruby/gem/ChangeLog | 0 ruby/gengem.sh | 3 ++- 5 files changed, 24 insertions(+), 7 deletions(-) delete mode 100644 ruby/gem/AUTHORS delete mode 100644 ruby/gem/ChangeLog diff --git a/Makefile.am b/Makefile.am index 3144972..d067fe0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -11,3 +11,20 @@ nobase_include_HEADERS = \ msgpack/unpack_define.h \ msgpack/unpack_template.h +EXTRA_DIST = \ + ruby/bench.rb \ + ruby/extconf.rb \ + ruby/gem/README \ + ruby/gem/Rakefile \ + ruby/gem/test/test_helper.rb \ + ruby/gengem.sh \ + ruby/msgpack.gemspec \ + ruby/pack.c \ + ruby/pack.h \ + ruby/rbinit.c \ + ruby/test_case.rb \ + ruby/test_format.rb \ + ruby/test_pack.rb \ + ruby/unpack.c \ + ruby/unpack.h + diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 406c57b..76770f4 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -20,12 +20,8 @@ nobase_include_HEADERS = \ msgpack/type/raw.hpp \ msgpack/type/tuple.hpp -noinst_HEADERS = \ - msgpack/type/tuple.hpp.erb - # FIXME -object.lo: msgpack/type/tuple.hpp -unpack.lo: msgpack/type/tuple.hpp msgpack/zone.hpp +object.lo: msgpack/type/tuple.hpp msgpack/zone.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb $(ERB) $< > $@.tmp @@ -39,6 +35,10 @@ MOSTLYCLEANFILES = \ msgpack/type/tuple.hpp \ msgpack/zone.hpp +EXTRA_DIST = \ + msgpack/type/tuple.hpp.erb \ + msgpack/zone.hpp.erb + libmsgpack_la_LIBADD = -L../c -lmsgpackc # -version-info CURRENT:REVISION:AGE diff --git a/ruby/gem/AUTHORS b/ruby/gem/AUTHORS deleted file mode 100644 index ababacb..0000000 --- a/ruby/gem/AUTHORS +++ /dev/null @@ -1 +0,0 @@ -FURUHASHI Sadayuki diff --git a/ruby/gem/ChangeLog b/ruby/gem/ChangeLog deleted file mode 100644 index e69de29..0000000 diff --git a/ruby/gengem.sh b/ruby/gengem.sh index 6c07be3..0afb8f5 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -9,7 +9,8 @@ cp rbinit.c gem/ext/ cp unpack.c gem/ext/ cp unpack.h gem/ext/ cat test_case.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FILE__) + '\/test_helper.rb'/" > gem/test/msgpack_test.rb -#cp ../README gem/README.txt +cp ../AUTHORS gem/AUTHORS +cp ../ChangeLog gem/ChangeLog cp ../msgpack/pack_define.h gem/msgpack/ cp ../msgpack/pack_template.h gem/msgpack/ cp ../msgpack/unpack_define.h gem/msgpack/ From c398d5d82820aaab89ed7e2e407db742376f5d3f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 13:46:20 +0900 Subject: [PATCH 0094/1172] fix packaging problem --- Makefile.am | 17 +++++++++++++++++ cpp/Makefile.am | 10 +++++----- ruby/gem/AUTHORS | 1 - ruby/gem/ChangeLog | 0 ruby/gengem.sh | 3 ++- 5 files changed, 24 insertions(+), 7 deletions(-) delete mode 100644 ruby/gem/AUTHORS delete mode 100644 ruby/gem/ChangeLog diff --git a/Makefile.am b/Makefile.am index 3144972..d067fe0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -11,3 +11,20 @@ nobase_include_HEADERS = \ msgpack/unpack_define.h \ msgpack/unpack_template.h +EXTRA_DIST = \ + ruby/bench.rb \ + ruby/extconf.rb \ + ruby/gem/README \ + ruby/gem/Rakefile \ + ruby/gem/test/test_helper.rb \ + ruby/gengem.sh \ + ruby/msgpack.gemspec \ + ruby/pack.c \ + ruby/pack.h \ + ruby/rbinit.c \ + ruby/test_case.rb \ + ruby/test_format.rb \ + ruby/test_pack.rb \ + ruby/unpack.c \ + ruby/unpack.h + diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 406c57b..76770f4 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -20,12 +20,8 @@ nobase_include_HEADERS = \ msgpack/type/raw.hpp \ msgpack/type/tuple.hpp -noinst_HEADERS = \ - msgpack/type/tuple.hpp.erb - # FIXME -object.lo: msgpack/type/tuple.hpp -unpack.lo: msgpack/type/tuple.hpp msgpack/zone.hpp +object.lo: msgpack/type/tuple.hpp msgpack/zone.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb $(ERB) $< > $@.tmp @@ -39,6 +35,10 @@ MOSTLYCLEANFILES = \ msgpack/type/tuple.hpp \ msgpack/zone.hpp +EXTRA_DIST = \ + msgpack/type/tuple.hpp.erb \ + msgpack/zone.hpp.erb + libmsgpack_la_LIBADD = -L../c -lmsgpackc # -version-info CURRENT:REVISION:AGE diff --git a/ruby/gem/AUTHORS b/ruby/gem/AUTHORS deleted file mode 100644 index ababacb..0000000 --- a/ruby/gem/AUTHORS +++ /dev/null @@ -1 +0,0 @@ -FURUHASHI Sadayuki diff --git a/ruby/gem/ChangeLog b/ruby/gem/ChangeLog deleted file mode 100644 index e69de29..0000000 diff --git a/ruby/gengem.sh b/ruby/gengem.sh index 6c07be3..0afb8f5 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -9,7 +9,8 @@ cp rbinit.c gem/ext/ cp unpack.c gem/ext/ cp unpack.h gem/ext/ cat test_case.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FILE__) + '\/test_helper.rb'/" > gem/test/msgpack_test.rb -#cp ../README gem/README.txt +cp ../AUTHORS gem/AUTHORS +cp ../ChangeLog gem/ChangeLog cp ../msgpack/pack_define.h gem/msgpack/ cp ../msgpack/pack_template.h gem/msgpack/ cp ../msgpack/unpack_define.h gem/msgpack/ From e7403013e5ca7d16d3610f81cb780ce4e0f548dc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 14:59:13 +0900 Subject: [PATCH 0095/1172] msgpack_zone: sizeof(int) bytes alignment --- c/zone.c | 19 ++----------------- c/zone.h | 26 +++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/c/zone.c b/c/zone.c index 7a275f6..877f4ed 100644 --- a/c/zone.c +++ b/c/zone.c @@ -19,7 +19,6 @@ #include #include - static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_size) { // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ @@ -74,25 +73,11 @@ static inline void clear_chunk_array(msgpack_zone_chunk_array* ca) ca->array[0].ptr = (char*)ca->array[0].alloc; } -void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) { msgpack_zone_chunk_array* const ca = &zone->chunk_array; - msgpack_zone_chunk* chunk = ca->tail; - - if(chunk->free > size) { - // chunkã«ç©ºã容é‡ãŒã‚ã‚‹ - // 空ã容é‡ã‚’消費ã—ã¦è¿”ã™ - - char* ptr = chunk->ptr; - - chunk->ptr += size; - chunk->free -= size; - - return ptr; - } - - chunk = ++ca->tail; + msgpack_zone_chunk* chunk = ++ca->tail; if(chunk == ca->end) { // ca->arrayã«ç©ºããŒãªã„ diff --git a/c/zone.h b/c/zone.h index 3c1188d..93a4642 100644 --- a/c/zone.h +++ b/c/zone.h @@ -65,7 +65,7 @@ void msgpack_zone_destroy(msgpack_zone* zone); msgpack_zone* msgpack_zone_new(size_t chunk_size); void msgpack_zone_free(msgpack_zone* zone); -void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); +static inline void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data); @@ -75,6 +75,30 @@ bool msgpack_zone_is_empty(msgpack_zone* zone); void msgpack_zone_clear(msgpack_zone* zone); + +#ifndef MSGPACK_ZONE_ALIGN +#define MSGPACK_ZONE_ALIGN sizeof(int) +#endif + +void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size); + +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +{ + size = ((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1); + + msgpack_zone_chunk* chunk = zone->chunk_array.tail; + + if(chunk->free > size) { + char* ptr = chunk->ptr; + chunk->ptr += size; + chunk->free -= size; + return ptr; + } + + return msgpack_zone_malloc_expand(zone, size); +} + + #ifdef __cplusplus } #endif From 319838e51757b59a9aa9e017491139602bcc1e68 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 14:59:13 +0900 Subject: [PATCH 0096/1172] msgpack_zone: sizeof(int) bytes alignment --- c/zone.c | 19 ++----------------- c/zone.h | 26 +++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/c/zone.c b/c/zone.c index 7a275f6..877f4ed 100644 --- a/c/zone.c +++ b/c/zone.c @@ -19,7 +19,6 @@ #include #include - static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_size) { // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ @@ -74,25 +73,11 @@ static inline void clear_chunk_array(msgpack_zone_chunk_array* ca) ca->array[0].ptr = (char*)ca->array[0].alloc; } -void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) { msgpack_zone_chunk_array* const ca = &zone->chunk_array; - msgpack_zone_chunk* chunk = ca->tail; - - if(chunk->free > size) { - // chunkã«ç©ºã容é‡ãŒã‚ã‚‹ - // 空ã容é‡ã‚’消費ã—ã¦è¿”ã™ - - char* ptr = chunk->ptr; - - chunk->ptr += size; - chunk->free -= size; - - return ptr; - } - - chunk = ++ca->tail; + msgpack_zone_chunk* chunk = ++ca->tail; if(chunk == ca->end) { // ca->arrayã«ç©ºããŒãªã„ diff --git a/c/zone.h b/c/zone.h index 3c1188d..93a4642 100644 --- a/c/zone.h +++ b/c/zone.h @@ -65,7 +65,7 @@ void msgpack_zone_destroy(msgpack_zone* zone); msgpack_zone* msgpack_zone_new(size_t chunk_size); void msgpack_zone_free(msgpack_zone* zone); -void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); +static inline void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data); @@ -75,6 +75,30 @@ bool msgpack_zone_is_empty(msgpack_zone* zone); void msgpack_zone_clear(msgpack_zone* zone); + +#ifndef MSGPACK_ZONE_ALIGN +#define MSGPACK_ZONE_ALIGN sizeof(int) +#endif + +void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size); + +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +{ + size = ((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1); + + msgpack_zone_chunk* chunk = zone->chunk_array.tail; + + if(chunk->free > size) { + char* ptr = chunk->ptr; + chunk->ptr += size; + chunk->free -= size; + return ptr; + } + + return msgpack_zone_malloc_expand(zone, size); +} + + #ifdef __cplusplus } #endif From e707b7a6000b34a80470638cfd66a6192578a97c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 15:49:24 +0900 Subject: [PATCH 0097/1172] zone::push_finalizer reverts memory allocation on exception --- c/zone.c | 58 ++++++++++++++++++++++-------------------------- c/zone.h | 37 ++++++++++++++++++++++++------ cpp/zone.hpp.erb | 23 ++++++++++++++++--- 3 files changed, 76 insertions(+), 42 deletions(-) diff --git a/c/zone.c b/c/zone.c index 877f4ed..1aaad9f 100644 --- a/c/zone.c +++ b/c/zone.c @@ -34,7 +34,7 @@ static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_s const size_t sz = chunk_size; char* ptr = (char*)malloc(sz); - if(!ptr) { + if(ptr == NULL) { free(array); return NULL; } @@ -88,7 +88,7 @@ void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) chunk = (msgpack_zone_chunk*)realloc(ca->array, sizeof(msgpack_zone_chunk) * nnext); - if(!chunk) { + if(chunk == NULL) { return NULL; } @@ -104,7 +104,7 @@ void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) } char* ptr = (char*)malloc(sz); - if(!ptr) { + if(ptr == NULL) { return NULL; } @@ -144,44 +144,38 @@ static inline void clear_finalizer_array(msgpack_zone_finalizer_array* fa) fa->tail = fa->array; } -bool msgpack_zone_push_finalizer(msgpack_zone* zone, +bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, void (*func)(void* data), void* data) { msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; - msgpack_zone_finalizer* fin = fa->tail; + const size_t nused = fa->end - fa->array; - if(fin == fa->end) { - // fa->arrayã«ç©ºããŒãªã„ - // fa->arrayã‚’æ‹¡å¼µã™ã‚‹ + size_t nnext; + if(nused == 0) { + // åˆå›žã®å‘¼ã³å‡ºã—：fa->tail == fa->end == fa->array == NULL - size_t nnext; - const size_t nused = fa->end - fa->array; + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + nnext = (sizeof(msgpack_zone_finalizer) < 72/2) ? + 72 / sizeof(msgpack_zone_finalizer) : 8; - if(nused == 0) { - // åˆå›žã®å‘¼ã³å‡ºã—：fa->tail == fa->end == fa->array == NULL - - // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ - nnext = (sizeof(msgpack_zone_finalizer) < 72/2) ? - 72 / sizeof(msgpack_zone_finalizer) : 8; - - } else { - nnext = (fa->end - fa->array) * 2; - } - - fin = (msgpack_zone_finalizer*)realloc(fa->array, - sizeof(msgpack_zone_finalizer) * nnext); - if(!fin) { - return false; - } - - fa->array = fin; - fa->end = fin + nnext; - fin = fa->tail = fin + nused; + } else { + nnext = nused * 2; } - fin->func = func; - fin->data = data; + msgpack_zone_finalizer* tmp = + (msgpack_zone_finalizer*)realloc(fa->array, + sizeof(msgpack_zone_finalizer) * nnext); + if(tmp == NULL) { + return false; + } + + fa->array = tmp; + fa->end = tmp + nnext; + fa->tail = tmp + nused; + + fa->tail->func = func; + fa->tail->data = data; ++fa->tail; diff --git a/c/zone.h b/c/zone.h index 93a4642..a3dfe41 100644 --- a/c/zone.h +++ b/c/zone.h @@ -67,7 +67,7 @@ void msgpack_zone_free(msgpack_zone* zone); static inline void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); -bool msgpack_zone_push_finalizer(msgpack_zone* zone, +static inline bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data); bool msgpack_zone_is_empty(msgpack_zone* zone); @@ -88,14 +88,37 @@ void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) msgpack_zone_chunk* chunk = zone->chunk_array.tail; - if(chunk->free > size) { - char* ptr = chunk->ptr; - chunk->ptr += size; - chunk->free -= size; - return ptr; + if(chunk->free < size) { + return msgpack_zone_malloc_expand(zone, size); } - return msgpack_zone_malloc_expand(zone, size); + char* ptr = chunk->ptr; + chunk->ptr += size; + chunk->free -= size; + + return ptr; +} + + +bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, + void (*func)(void* data), void* data); + +bool msgpack_zone_push_finalizer(msgpack_zone* zone, + void (*func)(void* data), void* data) +{ + msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; + msgpack_zone_finalizer* fin = fa->tail; + + if(fin == fa->end) { + return msgpack_zone_push_finalizer_expand(zone, func, data); + } + + fin->func = func; + fin->data = data; + + ++fa->tail; + + return true; } diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 930c8e8..874c900 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -45,6 +45,8 @@ public: <%}%> private: + void undo_malloc(size_t s); + template static void object_destructor(void* obj); @@ -91,14 +93,29 @@ void zone::object_destructor(void* obj) reinterpret_cast(obj)->~T(); } +inline void zone::undo_malloc(size_t s) +{ + msgpack_zone_chunk* chunk = chunk_array.tail; + chunk->ptr -= s; + chunk->free += s; +} + <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) { void* x = malloc(sizeof(T)); - push_finalizer(&zone::object_destructor, x); - try { return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); } - catch (...) { --finalizer_array.tail; throw; } + if(!msgpack_zone_push_finalizer(this, &zone::object_destructor, x)) { + undo_malloc(sizeof(T)); + throw std::bad_alloc(); + } + try { + return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); + } catch (...) { + --finalizer_array.tail; + undo_malloc(sizeof(T)); + throw; + } } <%}%> From 21040d9cd3b03ee27248b39a6b657948fdac5f9d Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 15:49:24 +0900 Subject: [PATCH 0098/1172] zone::push_finalizer reverts memory allocation on exception --- c/zone.c | 58 ++++++++++++++++++++++-------------------------- c/zone.h | 37 ++++++++++++++++++++++++------ cpp/zone.hpp.erb | 23 ++++++++++++++++--- 3 files changed, 76 insertions(+), 42 deletions(-) diff --git a/c/zone.c b/c/zone.c index 877f4ed..1aaad9f 100644 --- a/c/zone.c +++ b/c/zone.c @@ -34,7 +34,7 @@ static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_s const size_t sz = chunk_size; char* ptr = (char*)malloc(sz); - if(!ptr) { + if(ptr == NULL) { free(array); return NULL; } @@ -88,7 +88,7 @@ void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) chunk = (msgpack_zone_chunk*)realloc(ca->array, sizeof(msgpack_zone_chunk) * nnext); - if(!chunk) { + if(chunk == NULL) { return NULL; } @@ -104,7 +104,7 @@ void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) } char* ptr = (char*)malloc(sz); - if(!ptr) { + if(ptr == NULL) { return NULL; } @@ -144,44 +144,38 @@ static inline void clear_finalizer_array(msgpack_zone_finalizer_array* fa) fa->tail = fa->array; } -bool msgpack_zone_push_finalizer(msgpack_zone* zone, +bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, void (*func)(void* data), void* data) { msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; - msgpack_zone_finalizer* fin = fa->tail; + const size_t nused = fa->end - fa->array; - if(fin == fa->end) { - // fa->arrayã«ç©ºããŒãªã„ - // fa->arrayã‚’æ‹¡å¼µã™ã‚‹ + size_t nnext; + if(nused == 0) { + // åˆå›žã®å‘¼ã³å‡ºã—：fa->tail == fa->end == fa->array == NULL - size_t nnext; - const size_t nused = fa->end - fa->array; + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + nnext = (sizeof(msgpack_zone_finalizer) < 72/2) ? + 72 / sizeof(msgpack_zone_finalizer) : 8; - if(nused == 0) { - // åˆå›žã®å‘¼ã³å‡ºã—：fa->tail == fa->end == fa->array == NULL - - // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ - nnext = (sizeof(msgpack_zone_finalizer) < 72/2) ? - 72 / sizeof(msgpack_zone_finalizer) : 8; - - } else { - nnext = (fa->end - fa->array) * 2; - } - - fin = (msgpack_zone_finalizer*)realloc(fa->array, - sizeof(msgpack_zone_finalizer) * nnext); - if(!fin) { - return false; - } - - fa->array = fin; - fa->end = fin + nnext; - fin = fa->tail = fin + nused; + } else { + nnext = nused * 2; } - fin->func = func; - fin->data = data; + msgpack_zone_finalizer* tmp = + (msgpack_zone_finalizer*)realloc(fa->array, + sizeof(msgpack_zone_finalizer) * nnext); + if(tmp == NULL) { + return false; + } + + fa->array = tmp; + fa->end = tmp + nnext; + fa->tail = tmp + nused; + + fa->tail->func = func; + fa->tail->data = data; ++fa->tail; diff --git a/c/zone.h b/c/zone.h index 93a4642..a3dfe41 100644 --- a/c/zone.h +++ b/c/zone.h @@ -67,7 +67,7 @@ void msgpack_zone_free(msgpack_zone* zone); static inline void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); -bool msgpack_zone_push_finalizer(msgpack_zone* zone, +static inline bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data); bool msgpack_zone_is_empty(msgpack_zone* zone); @@ -88,14 +88,37 @@ void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) msgpack_zone_chunk* chunk = zone->chunk_array.tail; - if(chunk->free > size) { - char* ptr = chunk->ptr; - chunk->ptr += size; - chunk->free -= size; - return ptr; + if(chunk->free < size) { + return msgpack_zone_malloc_expand(zone, size); } - return msgpack_zone_malloc_expand(zone, size); + char* ptr = chunk->ptr; + chunk->ptr += size; + chunk->free -= size; + + return ptr; +} + + +bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, + void (*func)(void* data), void* data); + +bool msgpack_zone_push_finalizer(msgpack_zone* zone, + void (*func)(void* data), void* data) +{ + msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; + msgpack_zone_finalizer* fin = fa->tail; + + if(fin == fa->end) { + return msgpack_zone_push_finalizer_expand(zone, func, data); + } + + fin->func = func; + fin->data = data; + + ++fa->tail; + + return true; } diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 930c8e8..874c900 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -45,6 +45,8 @@ public: <%}%> private: + void undo_malloc(size_t s); + template static void object_destructor(void* obj); @@ -91,14 +93,29 @@ void zone::object_destructor(void* obj) reinterpret_cast(obj)->~T(); } +inline void zone::undo_malloc(size_t s) +{ + msgpack_zone_chunk* chunk = chunk_array.tail; + chunk->ptr -= s; + chunk->free += s; +} + <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) { void* x = malloc(sizeof(T)); - push_finalizer(&zone::object_destructor, x); - try { return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); } - catch (...) { --finalizer_array.tail; throw; } + if(!msgpack_zone_push_finalizer(this, &zone::object_destructor, x)) { + undo_malloc(sizeof(T)); + throw std::bad_alloc(); + } + try { + return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); + } catch (...) { + --finalizer_array.tail; + undo_malloc(sizeof(T)); + throw; + } } <%}%> From 2fad10ca3fb289c4aa20e1045fa89d2178ac1fde Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 16:40:02 +0900 Subject: [PATCH 0099/1172] zone::base --- cpp/zone.hpp.erb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 874c900..9e85080 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -45,11 +45,13 @@ public: <%}%> private: - void undo_malloc(size_t s); + void undo_malloc(size_t size); template static void object_destructor(void* obj); + typedef msgpack_zone base; + private: zone(const zone&); }; @@ -93,11 +95,11 @@ void zone::object_destructor(void* obj) reinterpret_cast(obj)->~T(); } -inline void zone::undo_malloc(size_t s) +inline void zone::undo_malloc(size_t size) { - msgpack_zone_chunk* chunk = chunk_array.tail; - chunk->ptr -= s; - chunk->free += s; + msgpack_zone_chunk* chunk = base::chunk_array.tail; + chunk->ptr -= size; + chunk->free += size; } <%0.upto(GENERATION_LIMIT) {|i|%> @@ -112,7 +114,7 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) try { return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); } catch (...) { - --finalizer_array.tail; + --base::finalizer_array.tail; undo_malloc(sizeof(T)); throw; } From 5fcf1d590ba3eff80b2a1f49db1641d058e1e3e5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 16:40:02 +0900 Subject: [PATCH 0100/1172] zone::base --- cpp/zone.hpp.erb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 874c900..9e85080 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -45,11 +45,13 @@ public: <%}%> private: - void undo_malloc(size_t s); + void undo_malloc(size_t size); template static void object_destructor(void* obj); + typedef msgpack_zone base; + private: zone(const zone&); }; @@ -93,11 +95,11 @@ void zone::object_destructor(void* obj) reinterpret_cast(obj)->~T(); } -inline void zone::undo_malloc(size_t s) +inline void zone::undo_malloc(size_t size) { - msgpack_zone_chunk* chunk = chunk_array.tail; - chunk->ptr -= s; - chunk->free += s; + msgpack_zone_chunk* chunk = base::chunk_array.tail; + chunk->ptr -= size; + chunk->free += size; } <%0.upto(GENERATION_LIMIT) {|i|%> @@ -112,7 +114,7 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) try { return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); } catch (...) { - --finalizer_array.tail; + --base::finalizer_array.tail; undo_malloc(sizeof(T)); throw; } From 09978e500a180e321b76328f4bb7e4f0dd12166f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 18:32:50 +0900 Subject: [PATCH 0101/1172] update ruby/msgpack.gemspec --- ruby/msgpack.gemspec | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index db2fcd5..bb5d9d0 100755 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,11 +1,12 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::CURRENT s.name = "msgpack" - s.version = "0.2.2" + s.version = "0.3.0" s.summary = "MessagePack" s.author = "FURUHASHI Sadayuki" - s.email = "frsyuki _at_ users.sourceforge.jp" - s.homepage = "https://launchpad.net/msgpack/" + s.email = "frsyuki@users.sourceforge.jp" + s.homepage = "http://msgpack.sourceforge.jp/" + s.rubyforge_project = "msgpack" s.require_paths = ["lib", "ext"] s.files = ["lib/**/*", "ext/**/*"].map {|g| Dir.glob(g) }.flatten end From d58d3ed2b0447709af70310a175336a560133573 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 18:32:50 +0900 Subject: [PATCH 0102/1172] update ruby/msgpack.gemspec --- ruby/msgpack.gemspec | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index db2fcd5..bb5d9d0 100755 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,11 +1,12 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::CURRENT s.name = "msgpack" - s.version = "0.2.2" + s.version = "0.3.0" s.summary = "MessagePack" s.author = "FURUHASHI Sadayuki" - s.email = "frsyuki _at_ users.sourceforge.jp" - s.homepage = "https://launchpad.net/msgpack/" + s.email = "frsyuki@users.sourceforge.jp" + s.homepage = "http://msgpack.sourceforge.jp/" + s.rubyforge_project = "msgpack" s.require_paths = ["lib", "ext"] s.files = ["lib/**/*", "ext/**/*"].map {|g| Dir.glob(g) }.flatten end From c7f193e4417cb253021bb902e4723d6bcf374c54 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:04:43 +0900 Subject: [PATCH 0103/1172] fix serialization bug on BigEndian environment --- msgpack/pack_template.h | 179 ++++++++++++++++++++++++++-------------- 1 file changed, 117 insertions(+), 62 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 212c0e6..14e3292 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -24,29 +24,79 @@ #endif #endif + #ifdef __LITTLE_ENDIAN__ -#define STORE_BE16(d) \ - ((char*)&d)[1], ((char*)&d)[0] +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] -#define STORE_BE32(d) \ - ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] -#define STORE_BE64(d) \ - ((char*)&d)[7], ((char*)&d)[6], ((char*)&d)[5], ((char*)&d)[4], \ - ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + #elif __BIG_ENDIAN__ -#define STORE_BE16(d) \ - ((char*)&d)[0], ((char*)&d)[1] +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] -#define STORE_BE32(d) \ - ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3] -#define STORE_BE64(d) \ - ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3], \ - ((char*)&d)[4], ((char*)&d)[5], ((char*)&d)[6], ((char*)&d)[7] +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[1] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[3] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[7] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] #endif @@ -71,10 +121,10 @@ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } while(0) @@ -83,14 +133,14 @@ do { \ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ } else if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -100,20 +150,20 @@ do { \ if(d < (1<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -124,24 +174,24 @@ do { \ if(d < (1ULL<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -151,11 +201,11 @@ do { \ do { \ if(d < -(1<<5)) { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, d}; \ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ } \ } while(0) @@ -164,24 +214,24 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -192,32 +242,32 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<15)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -229,46 +279,46 @@ do { \ if(d < -(1LL<<15)) { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; \ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ } else { \ if(d < (1LL<<16)) { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -554,7 +604,7 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { union { char buf[4]; uint32_t num; } f; *((float*)&f.buf) = d; // FIXME - const unsigned char buf[5] = {0xca, STORE_BE32(f.num)}; + const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; msgpack_pack_append_buffer(x, buf, 5); } @@ -562,7 +612,7 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { union { char buf[8]; uint64_t num; } f; *((double*)&f.buf) = d; // FIXME - const unsigned char buf[9] = {0xcb, STORE_BE64(f.num)}; + const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -606,11 +656,11 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xdc, STORE_BE16(d)}; + unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdd, STORE_BE32(d)}; + unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -627,11 +677,11 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xde, STORE_BE16(d)}; + unsigned char buf[3] = {0xde, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdf, STORE_BE32(d)}; + unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -648,11 +698,11 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) msgpack_pack_append_buffer(x, &d, 1); } else if(l < 65536) { uint16_t d = (uint16_t)l; - unsigned char buf[3] = {0xda, STORE_BE16(d)}; + unsigned char buf[3] = {0xda, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)l; - unsigned char buf[5] = {0xdb, STORE_BE32(d)}; + unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -666,9 +716,14 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer -#undef STORE_BE16 -#undef STORE_BE32 -#undef STORE_BE64 +#undef STORE16_BE16 + +#undef STORE32_BE16 +#undef STORE32_BE32 + +#undef STORE64_BE16 +#undef STORE64_BE32 +#undef STORE64_BE64 #undef msgpack_pack_real_uint8 #undef msgpack_pack_real_uint16 From 44cdc5f9763d2fd119774bbd026ba635eb04e2e5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:04:43 +0900 Subject: [PATCH 0104/1172] fix serialization bug on BigEndian environment --- msgpack/pack_template.h | 179 ++++++++++++++++++++++++++-------------- 1 file changed, 117 insertions(+), 62 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 212c0e6..14e3292 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -24,29 +24,79 @@ #endif #endif + #ifdef __LITTLE_ENDIAN__ -#define STORE_BE16(d) \ - ((char*)&d)[1], ((char*)&d)[0] +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] -#define STORE_BE32(d) \ - ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] -#define STORE_BE64(d) \ - ((char*)&d)[7], ((char*)&d)[6], ((char*)&d)[5], ((char*)&d)[4], \ - ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + #elif __BIG_ENDIAN__ -#define STORE_BE16(d) \ - ((char*)&d)[0], ((char*)&d)[1] +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] -#define STORE_BE32(d) \ - ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3] -#define STORE_BE64(d) \ - ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3], \ - ((char*)&d)[4], ((char*)&d)[5], ((char*)&d)[6], ((char*)&d)[7] +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[1] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[3] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[7] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] #endif @@ -71,10 +121,10 @@ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } while(0) @@ -83,14 +133,14 @@ do { \ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ } else if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -100,20 +150,20 @@ do { \ if(d < (1<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -124,24 +174,24 @@ do { \ if(d < (1ULL<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -151,11 +201,11 @@ do { \ do { \ if(d < -(1<<5)) { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, d}; \ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ } \ } while(0) @@ -164,24 +214,24 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -192,32 +242,32 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<15)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -229,46 +279,46 @@ do { \ if(d < -(1LL<<15)) { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; \ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ } else { \ if(d < (1LL<<16)) { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -554,7 +604,7 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { union { char buf[4]; uint32_t num; } f; *((float*)&f.buf) = d; // FIXME - const unsigned char buf[5] = {0xca, STORE_BE32(f.num)}; + const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; msgpack_pack_append_buffer(x, buf, 5); } @@ -562,7 +612,7 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { union { char buf[8]; uint64_t num; } f; *((double*)&f.buf) = d; // FIXME - const unsigned char buf[9] = {0xcb, STORE_BE64(f.num)}; + const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -606,11 +656,11 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xdc, STORE_BE16(d)}; + unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdd, STORE_BE32(d)}; + unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -627,11 +677,11 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xde, STORE_BE16(d)}; + unsigned char buf[3] = {0xde, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdf, STORE_BE32(d)}; + unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -648,11 +698,11 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) msgpack_pack_append_buffer(x, &d, 1); } else if(l < 65536) { uint16_t d = (uint16_t)l; - unsigned char buf[3] = {0xda, STORE_BE16(d)}; + unsigned char buf[3] = {0xda, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)l; - unsigned char buf[5] = {0xdb, STORE_BE32(d)}; + unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -666,9 +716,14 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer -#undef STORE_BE16 -#undef STORE_BE32 -#undef STORE_BE64 +#undef STORE16_BE16 + +#undef STORE32_BE16 +#undef STORE32_BE32 + +#undef STORE64_BE16 +#undef STORE64_BE32 +#undef STORE64_BE64 #undef msgpack_pack_real_uint8 #undef msgpack_pack_real_uint16 From 163fc6f589b9e0de89ae091b3a5cb0b0bffdf71e Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:15:22 +0900 Subject: [PATCH 0105/1172] fix serialization bug on BigEndian environment 3 --- msgpack/pack_template.h | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 14e3292..aa620f5 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -330,49 +330,49 @@ do { \ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) { - const unsigned char buf[2] = {0xcc, d}; + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) { - const unsigned char buf[2] = {0xd0, d}; + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { - const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -674,7 +674,7 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; - msgpack_pack_append_buffer(x, &d, 1); + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; unsigned char buf[3] = {0xde, STORE16_BE16(d)}; @@ -695,7 +695,7 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; - msgpack_pack_append_buffer(x, &d, 1); + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); } else if(l < 65536) { uint16_t d = (uint16_t)l; unsigned char buf[3] = {0xda, STORE16_BE16(d)}; @@ -716,11 +716,16 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer +#undef STORE8_BE8 + +#undef STORE16_BE8 #undef STORE16_BE16 +#undef STORE32_BE8 #undef STORE32_BE16 #undef STORE32_BE32 +#undef STORE64_BE8 #undef STORE64_BE16 #undef STORE64_BE32 #undef STORE64_BE64 From c612a177ccf7efe5ad3a7f67269b1399d4e8f96c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:15:22 +0900 Subject: [PATCH 0106/1172] fix serialization bug on BigEndian environment 3 --- msgpack/pack_template.h | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 14e3292..aa620f5 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -330,49 +330,49 @@ do { \ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) { - const unsigned char buf[2] = {0xcc, d}; + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) { - const unsigned char buf[2] = {0xd0, d}; + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { - const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -674,7 +674,7 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; - msgpack_pack_append_buffer(x, &d, 1); + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; unsigned char buf[3] = {0xde, STORE16_BE16(d)}; @@ -695,7 +695,7 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; - msgpack_pack_append_buffer(x, &d, 1); + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); } else if(l < 65536) { uint16_t d = (uint16_t)l; unsigned char buf[3] = {0xda, STORE16_BE16(d)}; @@ -716,11 +716,16 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer +#undef STORE8_BE8 + +#undef STORE16_BE8 #undef STORE16_BE16 +#undef STORE32_BE8 #undef STORE32_BE16 #undef STORE32_BE32 +#undef STORE64_BE8 #undef STORE64_BE16 #undef STORE64_BE32 #undef STORE64_BE64 From cc7379d532b5bd9671e05860207dde0970d6e270 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:36:56 +0900 Subject: [PATCH 0107/1172] more test caces --- ruby/test_case.rb | 78 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/ruby/test_case.rb b/ruby/test_case.rb index 1a06b10..2d897df 100644 --- a/ruby/test_case.rb +++ b/ruby/test_case.rb @@ -113,6 +113,80 @@ class MessagePackTestFormat < Test::Unit::TestCase #check_array 5, (1<<32)-1 # memory error end + it "nil" do + match nil, "\xc0" + end + + it "false" do + match false, "\xc2" + end + + it "true" do + match true, "\xc3" + end + + it "0" do + match 0, "\x00" + end + + it "127" do + match 127, "\x7f" + end + + it "128" do + match 128, "\xcc\x80" + end + + it "256" do + match 256, "\xcd\x01\x00" + end + + it "-1" do + match -1, "\xff" + end + + it "-33" do + match -33, "\xd0\xdf" + end + + it "-129" do + match -129, "\xd1\xff\x7f" + end + + it "{1=>1}" do + match ({1=>1}), "\x81\x01\x01" + end + + it "1.0" do + match 1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00" + end + + it "[]" do + match [], "\x90" + end + + it "[0, 1, ..., 14]" do + match (0..14).to_a, "\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e" + end + + it "[0, 1, ..., 15]" do + match (0..15).to_a, "\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + end + + it "{}" do + match ({}), "\x80" + end + + it "{0=>0, 1=>1, ..., 14=>14}" do + a = (0..14).to_a; + match Hash[*a.zip(a).flatten], "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a" + end + + it "{0=>0, 1=>1, ..., 15=>15}" do + a = (0..15).to_a; + match Hash[*a.zip(a).flatten], "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a" + end + # it "fixmap" do # check_map 1, 0 # check_map 1, (1<<4)-1 @@ -143,8 +217,8 @@ class MessagePackTestFormat < Test::Unit::TestCase check num+overhead, Array.new(num) end - def check_map(overhead, num) - # FIXME + def match(obj, buf) + assert_equal(obj.to_msgpack, buf) end end From 840388720ef19e160e970e25329e176328b74fd0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:36:56 +0900 Subject: [PATCH 0108/1172] more test caces --- ruby/test_case.rb | 78 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/ruby/test_case.rb b/ruby/test_case.rb index 1a06b10..2d897df 100644 --- a/ruby/test_case.rb +++ b/ruby/test_case.rb @@ -113,6 +113,80 @@ class MessagePackTestFormat < Test::Unit::TestCase #check_array 5, (1<<32)-1 # memory error end + it "nil" do + match nil, "\xc0" + end + + it "false" do + match false, "\xc2" + end + + it "true" do + match true, "\xc3" + end + + it "0" do + match 0, "\x00" + end + + it "127" do + match 127, "\x7f" + end + + it "128" do + match 128, "\xcc\x80" + end + + it "256" do + match 256, "\xcd\x01\x00" + end + + it "-1" do + match -1, "\xff" + end + + it "-33" do + match -33, "\xd0\xdf" + end + + it "-129" do + match -129, "\xd1\xff\x7f" + end + + it "{1=>1}" do + match ({1=>1}), "\x81\x01\x01" + end + + it "1.0" do + match 1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00" + end + + it "[]" do + match [], "\x90" + end + + it "[0, 1, ..., 14]" do + match (0..14).to_a, "\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e" + end + + it "[0, 1, ..., 15]" do + match (0..15).to_a, "\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + end + + it "{}" do + match ({}), "\x80" + end + + it "{0=>0, 1=>1, ..., 14=>14}" do + a = (0..14).to_a; + match Hash[*a.zip(a).flatten], "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a" + end + + it "{0=>0, 1=>1, ..., 15=>15}" do + a = (0..15).to_a; + match Hash[*a.zip(a).flatten], "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a" + end + # it "fixmap" do # check_map 1, 0 # check_map 1, (1<<4)-1 @@ -143,8 +217,8 @@ class MessagePackTestFormat < Test::Unit::TestCase check num+overhead, Array.new(num) end - def check_map(overhead, num) - # FIXME + def match(obj, buf) + assert_equal(obj.to_msgpack, buf) end end From b8cc8b72bd896066d02aa869405161375b528b33 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:54:07 +0900 Subject: [PATCH 0109/1172] version 0.3.1 --- configure.in | 2 +- ruby/gem/Rakefile | 2 +- ruby/msgpack.gemspec | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index 2c1bee7..3569244 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.0) +AM_INIT_AUTOMAKE(msgpack, 0.3.1) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/ruby/gem/Rakefile b/ruby/gem/Rakefile index 87635cb..5445906 100644 --- a/ruby/gem/Rakefile +++ b/ruby/gem/Rakefile @@ -17,7 +17,7 @@ DESCRIPTION = "Binary-based efficient data interchange format." RUBYFORGE_PROJECT = "msgpack" HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org" BIN_FILES = %w( ) -VERS = "0.3.0" +VERS = "0.3.1" #REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil REV = nil diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index bb5d9d0..f0b5c44 100755 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::CURRENT s.name = "msgpack" - s.version = "0.3.0" + s.version = "0.3.1" s.summary = "MessagePack" s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" From 5bd53f018f5317b96e635bbfec6a42b7950d1ef0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:54:07 +0900 Subject: [PATCH 0110/1172] version 0.3.1 --- configure.in | 2 +- ruby/gem/Rakefile | 2 +- ruby/msgpack.gemspec | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index 2c1bee7..3569244 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.0) +AM_INIT_AUTOMAKE(msgpack, 0.3.1) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/ruby/gem/Rakefile b/ruby/gem/Rakefile index 87635cb..5445906 100644 --- a/ruby/gem/Rakefile +++ b/ruby/gem/Rakefile @@ -17,7 +17,7 @@ DESCRIPTION = "Binary-based efficient data interchange format." RUBYFORGE_PROJECT = "msgpack" HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org" BIN_FILES = %w( ) -VERS = "0.3.0" +VERS = "0.3.1" #REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil REV = nil diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index bb5d9d0..f0b5c44 100755 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::CURRENT s.name = "msgpack" - s.version = "0.3.0" + s.version = "0.3.1" s.summary = "MessagePack" s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" From 8c62d93bcad27621a02fcc9923cb8e6ce46f7c9a Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 12:55:41 +0900 Subject: [PATCH 0111/1172] added perl support --- perl/Makefile.PL | 34 + perl/MessagePack.c | 39 + perl/benchmark/p1.pl | 16 + perl/lib/Data/MessagePack.pm | 33 + perl/lib/Data/MessagePack/Unpacker.pod | 52 + perl/pack.c | 122 + perl/ppport.h | 6984 ++++++++++++++++++++++++ perl/t/00_compile.t | 6 + perl/t/01_pack.t | 61 + perl/t/02_unpack.t | 24 + perl/t/03_stream_unpack.t | 32 + perl/t/04_invert.t | 24 + perl/t/Util.pm | 20 + perl/t/data.pl | 15 + perl/unpack.c | 268 + perl/xt/99_pod.t | 4 + 16 files changed, 7734 insertions(+) create mode 100644 perl/Makefile.PL create mode 100644 perl/MessagePack.c create mode 100644 perl/benchmark/p1.pl create mode 100644 perl/lib/Data/MessagePack.pm create mode 100644 perl/lib/Data/MessagePack/Unpacker.pod create mode 100644 perl/pack.c create mode 100644 perl/ppport.h create mode 100644 perl/t/00_compile.t create mode 100644 perl/t/01_pack.t create mode 100644 perl/t/02_unpack.t create mode 100644 perl/t/03_stream_unpack.t create mode 100644 perl/t/04_invert.t create mode 100644 perl/t/Util.pm create mode 100644 perl/t/data.pl create mode 100644 perl/unpack.c create mode 100644 perl/xt/99_pod.t diff --git a/perl/Makefile.PL b/perl/Makefile.PL new file mode 100644 index 0000000..f60e125 --- /dev/null +++ b/perl/Makefile.PL @@ -0,0 +1,34 @@ +use inc::Module::Install; +name 'Data-MessagePack'; +all_from 'lib/Data/MessagePack.pm'; + +perl_version '5.008005'; +license 'perl'; +can_cc or die "This module requires a C compiler"; + +my $ccflags = '-I../ '; + +makemaker_args( + OBJECT => '$(O_FILES)', + LIBS => [''], + CCFLAGS => $ccflags, + clean => { + FILES => q{ + *.stackdump + *.gcov *.gcda *.gcno + *.out + nytprof + cover_db + }, + }, +); + +tests 't/*.t'; +author_tests('xt'); + +auto_set_repository; +build_requires 'Test::More'; +use_test_base; +auto_include; +WriteAll; + diff --git a/perl/MessagePack.c b/perl/MessagePack.c new file mode 100644 index 0000000..c40e46a --- /dev/null +++ b/perl/MessagePack.c @@ -0,0 +1,39 @@ +#ifdef __cplusplus +extern "C" { +#endif +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "ppport.h" +#ifdef __cplusplus +}; +#endif + +XS(xs_pack); +XS(xs_unpack); +XS(xs_unpacker_new); +XS(xs_unpacker_execute); +XS(xs_unpacker_execute_limit); +XS(xs_unpacker_is_finished); +XS(xs_unpacker_data); +XS(xs_unpacker_reset); + +XS(boot_Data__MessagePack) { + dXSARGS; + HV * stash; + + newXS("Data::MessagePack::pack", xs_pack, __FILE__); + newXS("Data::MessagePack::unpack", xs_unpack, __FILE__); + stash = gv_stashpvn("Data::MessagePack", strlen("Data::MessagePack"), TRUE); + newCONSTSUB(stash, "true", &PL_sv_yes); + newCONSTSUB(stash, "false", &PL_sv_no); + + newXS("Data::MessagePack::Unpacker::new", xs_unpacker_new, __FILE__); + newXS("Data::MessagePack::Unpacker::execute", xs_unpacker_execute, __FILE__); + newXS("Data::MessagePack::Unpacker::execute_limit", xs_unpacker_execute_limit, __FILE__); + newXS("Data::MessagePack::Unpacker::is_finished", xs_unpacker_is_finished, __FILE__); + newXS("Data::MessagePack::Unpacker::data", xs_unpacker_data, __FILE__); + newXS("Data::MessagePack::Unpacker::reset", xs_unpacker_reset, __FILE__); + +} + diff --git a/perl/benchmark/p1.pl b/perl/benchmark/p1.pl new file mode 100644 index 0000000..257932e --- /dev/null +++ b/perl/benchmark/p1.pl @@ -0,0 +1,16 @@ +use strict; +use warnings; +use Data::MessagePack; +use JSON::XS; +use Benchmark ':all'; + +my $a = [0..2**24]; + +print "-- serialize\n"; +cmpthese( + -1 => { + json => sub { JSON::XS::encode_json($a) }, + mp => sub { Data::MessagePack->pack($a) }, + } +); + diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm new file mode 100644 index 0000000..d26bf9e --- /dev/null +++ b/perl/lib/Data/MessagePack.pm @@ -0,0 +1,33 @@ +package Data::MessagePack; +use strict; +use warnings; +use XSLoader; + +our $VERSION = 0.01; + +XSLoader::load(__PACKAGE__, $VERSION); + +1; +__END__ + +=head1 NAME + +Data::MessagePack - messagepack + +=head1 SYNOPSIS + + my $packed = Data::MessagePack->pack($dat); + my $unpacked = Data::MessagePack->unpack($dat); + +=head1 DESCRIPTION + +Data::MessagePack is a binary packer for perl. + +=head1 AUTHORS + +Tokuhiro Matsuno + +=head1 SEE ALSO + +L + diff --git a/perl/lib/Data/MessagePack/Unpacker.pod b/perl/lib/Data/MessagePack/Unpacker.pod new file mode 100644 index 0000000..61cbd21 --- /dev/null +++ b/perl/lib/Data/MessagePack/Unpacker.pod @@ -0,0 +1,52 @@ +=head1 NAME + +Data::MessagePack::Unpacker - messagepack streaming deserializer + +=head1 SYNOPSIS + + use Data::Dumper; + my $up = Data::MessagePack::Unpacker->new; + my $ret = $up->execute($v, 0); + if ($ret != length($v)) { + fail "extra bytes"; + } + return Dumper($up->data); + +=head1 DESCRIPTION + +This is an streaming deserializer for messagepack. + +=head1 METHODS + +=over 4 + +=item my $up = Data::MessagePack::Unpacker->new() + +create new stream deserializer + +=item $up->execute() + +=item $up->execute_limit() + +=item $up->is_finished() + +is this deserializer finished? + +=item $up->data() + +returns deserialized object. + +=item $up->reset() + +reset the stream deserializer, without memory zone. + +=back + +=head1 AUTHORS + +Tokuhiro Matsuno + +=head1 SEE ALSO + +L + diff --git a/perl/pack.c b/perl/pack.c new file mode 100644 index 0000000..5bb667b --- /dev/null +++ b/perl/pack.c @@ -0,0 +1,122 @@ +#ifdef __cplusplus +extern "C" { +#endif +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "ppport.h" +#ifdef __cplusplus +}; +#endif + +#include "msgpack/pack_define.h" + +#define msgpack_pack_inline_func(name) \ + static inline void msgpack_pack ## name + +#define msgpack_pack_inline_func_cint(name) \ + static inline void msgpack_pack ## name + +#define msgpack_pack_user SV* + +#define msgpack_pack_append_buffer(user, buf, len) \ + sv_catpvn(user, (const char*)(buf), len); + +#include "msgpack/pack_template.h" + +#define _PACK_WRAPPER(t) msgpack_pack_##t +#define PACK_WRAPPER(t) _PACK_WRAPPER(t) + +// move to pack.c +static void _msgpack_pack_sv(SV* buf, SV* val) { + if (val==NULL) { + msgpack_pack_nil(buf); + return; + } + + switch (SvTYPE(val)) { + case SVt_NULL: + msgpack_pack_nil(buf); + break; + case SVt_IV: + if (SvIOK_UV(val)) { + msgpack_pack_uint32(buf, SvUV(val)); + } else { + PACK_WRAPPER(IVTYPE)(buf, SvIV(val)); + } + break; + case SVt_PVNV: + { + STRLEN len = 0; + char *pv = SvPV(val, len); + if (len == 1 && *pv == '1') { + msgpack_pack_true(buf); + } else if (len == 0 && *pv==0) { + msgpack_pack_false(buf); + } else { + msgpack_pack_nil(buf); + } + } + break; + case SVt_PV: + { + STRLEN len; + char * cval = SvPV(val, len); + msgpack_pack_raw(buf, len); + msgpack_pack_raw_body(buf, cval, len); + } + break; + case SVt_NV: + PACK_WRAPPER(NVTYPE)(buf, SvNV(val)); + break; + case SVt_PVAV: + { + AV* ary = (AV*)val; + int len = av_len(ary) + 1; + int i; + msgpack_pack_array(buf, len); + for (i=0; i is designed to support operation with Perl +installations back to 5.003, and has been tested up to 5.10.0. + +=head1 OPTIONS + +=head2 --help + +Display a brief usage summary. + +=head2 --version + +Display the version of F. + +=head2 --patch=I + +If this option is given, a single patch file will be created if +any changes are suggested. This requires a working diff program +to be installed on your system. + +=head2 --copy=I + +If this option is given, a copy of each file will be saved with +the given suffix that contains the suggested changes. This does +not require any external programs. Note that this does not +automagially add a dot between the original filename and the +suffix. If you want the dot, you have to include it in the option +argument. + +If neither C<--patch> or C<--copy> are given, the default is to +simply print the diffs for each file. This requires either +C or a C program to be installed. + +=head2 --diff=I + +Manually set the diff program and options to use. The default +is to use C, when installed, and output unified +context diffs. + +=head2 --compat-version=I + +Tell F to check for compatibility with the given +Perl version. The default is to check for compatibility with Perl +version 5.003. You can use this option to reduce the output +of F if you intend to be backward compatible only +down to a certain Perl version. + +=head2 --cplusplus + +Usually, F will detect C++ style comments and +replace them with C style comments for portability reasons. +Using this option instructs F to leave C++ +comments untouched. + +=head2 --quiet + +Be quiet. Don't print anything except fatal errors. + +=head2 --nodiag + +Don't output any diagnostic messages. Only portability +alerts will be printed. + +=head2 --nohints + +Don't output any hints. Hints often contain useful portability +notes. Warnings will still be displayed. + +=head2 --nochanges + +Don't suggest any changes. Only give diagnostic output and hints +unless these are also deactivated. + +=head2 --nofilter + +Don't filter the list of input files. By default, files not looking +like source code (i.e. not *.xs, *.c, *.cc, *.cpp or *.h) are skipped. + +=head2 --strip + +Strip all script and documentation functionality from F. +This reduces the size of F dramatically and may be useful +if you want to include F in smaller modules without +increasing their distribution size too much. + +The stripped F will have a C<--unstrip> option that allows +you to undo the stripping, but only if an appropriate C +module is installed. + +=head2 --list-provided + +Lists the API elements for which compatibility is provided by +F. Also lists if it must be explicitly requested, +if it has dependencies, and if there are hints or warnings for it. + +=head2 --list-unsupported + +Lists the API elements that are known not to be supported by +F and below which version of Perl they probably +won't be available or work. + +=head2 --api-info=I + +Show portability information for API elements matching I. +If I is surrounded by slashes, it is interpreted as a regular +expression. + +=head1 DESCRIPTION + +In order for a Perl extension (XS) module to be as portable as possible +across differing versions of Perl itself, certain steps need to be taken. + +=over 4 + +=item * + +Including this header is the first major one. This alone will give you +access to a large part of the Perl API that hasn't been available in +earlier Perl releases. Use + + perl ppport.h --list-provided + +to see which API elements are provided by ppport.h. + +=item * + +You should avoid using deprecated parts of the API. For example, using +global Perl variables without the C prefix is deprecated. Also, +some API functions used to have a C prefix. Using this form is +also deprecated. You can safely use the supported API, as F +will provide wrappers for older Perl versions. + +=item * + +If you use one of a few functions or variables that were not present in +earlier versions of Perl, and that can't be provided using a macro, you +have to explicitly request support for these functions by adding one or +more C<#define>s in your source code before the inclusion of F. + +These functions or variables will be marked C in the list shown +by C<--list-provided>. + +Depending on whether you module has a single or multiple files that +use such functions or variables, you want either C or global +variants. + +For a C function or variable (used only in a single source +file), use: + + #define NEED_function + #define NEED_variable + +For a global function or variable (used in multiple source files), +use: + + #define NEED_function_GLOBAL + #define NEED_variable_GLOBAL + +Note that you mustn't have more than one global request for the +same function or variable in your project. + + Function / Variable Static Request Global Request + ----------------------------------------------------------------------------------------- + PL_parser NEED_PL_parser NEED_PL_parser_GLOBAL + PL_signals NEED_PL_signals NEED_PL_signals_GLOBAL + eval_pv() NEED_eval_pv NEED_eval_pv_GLOBAL + grok_bin() NEED_grok_bin NEED_grok_bin_GLOBAL + grok_hex() NEED_grok_hex NEED_grok_hex_GLOBAL + grok_number() NEED_grok_number NEED_grok_number_GLOBAL + grok_numeric_radix() NEED_grok_numeric_radix NEED_grok_numeric_radix_GLOBAL + grok_oct() NEED_grok_oct NEED_grok_oct_GLOBAL + load_module() NEED_load_module NEED_load_module_GLOBAL + my_snprintf() NEED_my_snprintf NEED_my_snprintf_GLOBAL + my_sprintf() NEED_my_sprintf NEED_my_sprintf_GLOBAL + my_strlcat() NEED_my_strlcat NEED_my_strlcat_GLOBAL + my_strlcpy() NEED_my_strlcpy NEED_my_strlcpy_GLOBAL + newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL + newRV_noinc() NEED_newRV_noinc NEED_newRV_noinc_GLOBAL + newSVpvn_flags() NEED_newSVpvn_flags NEED_newSVpvn_flags_GLOBAL + newSVpvn_share() NEED_newSVpvn_share NEED_newSVpvn_share_GLOBAL + pv_display() NEED_pv_display NEED_pv_display_GLOBAL + pv_escape() NEED_pv_escape NEED_pv_escape_GLOBAL + pv_pretty() NEED_pv_pretty NEED_pv_pretty_GLOBAL + sv_2pv_flags() NEED_sv_2pv_flags NEED_sv_2pv_flags_GLOBAL + sv_2pvbyte() NEED_sv_2pvbyte NEED_sv_2pvbyte_GLOBAL + sv_catpvf_mg() NEED_sv_catpvf_mg NEED_sv_catpvf_mg_GLOBAL + sv_catpvf_mg_nocontext() NEED_sv_catpvf_mg_nocontext NEED_sv_catpvf_mg_nocontext_GLOBAL + sv_pvn_force_flags() NEED_sv_pvn_force_flags NEED_sv_pvn_force_flags_GLOBAL + sv_setpvf_mg() NEED_sv_setpvf_mg NEED_sv_setpvf_mg_GLOBAL + sv_setpvf_mg_nocontext() NEED_sv_setpvf_mg_nocontext NEED_sv_setpvf_mg_nocontext_GLOBAL + vload_module() NEED_vload_module NEED_vload_module_GLOBAL + vnewSVpvf() NEED_vnewSVpvf NEED_vnewSVpvf_GLOBAL + warner() NEED_warner NEED_warner_GLOBAL + +To avoid namespace conflicts, you can change the namespace of the +explicitly exported functions / variables using the C +macro. Just C<#define> the macro before including C: + + #define DPPP_NAMESPACE MyOwnNamespace_ + #include "ppport.h" + +The default namespace is C. + +=back + +The good thing is that most of the above can be checked by running +F on your source code. See the next section for +details. + +=head1 EXAMPLES + +To verify whether F is needed for your module, whether you +should make any changes to your code, and whether any special defines +should be used, F can be run as a Perl script to check your +source code. Simply say: + + perl ppport.h + +The result will usually be a list of patches suggesting changes +that should at least be acceptable, if not necessarily the most +efficient solution, or a fix for all possible problems. + +If you know that your XS module uses features only available in +newer Perl releases, if you're aware that it uses C++ comments, +and if you want all suggestions as a single patch file, you could +use something like this: + + perl ppport.h --compat-version=5.6.0 --cplusplus --patch=test.diff + +If you only want your code to be scanned without any suggestions +for changes, use: + + perl ppport.h --nochanges + +You can specify a different C program or options, using +the C<--diff> option: + + perl ppport.h --diff='diff -C 10' + +This would output context diffs with 10 lines of context. + +If you want to create patched copies of your files instead, use: + + perl ppport.h --copy=.new + +To display portability information for the C function, +use: + + perl ppport.h --api-info=newSVpvn + +Since the argument to C<--api-info> can be a regular expression, +you can use + + perl ppport.h --api-info=/_nomg$/ + +to display portability information for all C<_nomg> functions or + + perl ppport.h --api-info=/./ + +to display information for all known API elements. + +=head1 BUGS + +If this version of F is causing failure during +the compilation of this module, please check if newer versions +of either this module or C are available on CPAN +before sending a bug report. + +If F was generated using the latest version of +C and is causing failure of this module, please +file a bug report using the CPAN Request Tracker at L. + +Please include the following information: + +=over 4 + +=item 1. + +The complete output from running "perl -V" + +=item 2. + +This file. + +=item 3. + +The name and version of the module you were trying to build. + +=item 4. + +A full log of the build that failed. + +=item 5. + +Any other information that you think could be relevant. + +=back + +For the latest version of this code, please get the C +module from CPAN. + +=head1 COPYRIGHT + +Version 3.x, Copyright (c) 2004-2009, Marcus Holland-Moritz. + +Version 2.x, Copyright (C) 2001, Paul Marquess. + +Version 1.x, Copyright (C) 1999, Kenneth Albanowski. + +This program is free software; you can redistribute it and/or +modify it under the same terms as Perl itself. + +=head1 SEE ALSO + +See L. + +=cut + +use strict; + +# Disable broken TRIE-optimization +BEGIN { eval '${^RE_TRIE_MAXBUF} = -1' if $] >= 5.009004 && $] <= 5.009005 } + +my $VERSION = 3.17; + +my %opt = ( + quiet => 0, + diag => 1, + hints => 1, + changes => 1, + cplusplus => 0, + filter => 1, + strip => 0, + version => 0, +); + +my($ppport) = $0 =~ /([\w.]+)$/; +my $LF = '(?:\r\n|[\r\n])'; # line feed +my $HS = "[ \t]"; # horizontal whitespace + +# Never use C comments in this file! +my $ccs = '/'.'*'; +my $cce = '*'.'/'; +my $rccs = quotemeta $ccs; +my $rcce = quotemeta $cce; + +eval { + require Getopt::Long; + Getopt::Long::GetOptions(\%opt, qw( + help quiet diag! filter! hints! changes! cplusplus strip version + patch=s copy=s diff=s compat-version=s + list-provided list-unsupported api-info=s + )) or usage(); +}; + +if ($@ and grep /^-/, @ARGV) { + usage() if "@ARGV" =~ /^--?h(?:elp)?$/; + die "Getopt::Long not found. Please don't use any options.\n"; +} + +if ($opt{version}) { + print "This is $0 $VERSION.\n"; + exit 0; +} + +usage() if $opt{help}; +strip() if $opt{strip}; + +if (exists $opt{'compat-version'}) { + my($r,$v,$s) = eval { parse_version($opt{'compat-version'}) }; + if ($@) { + die "Invalid version number format: '$opt{'compat-version'}'\n"; + } + die "Only Perl 5 is supported\n" if $r != 5; + die "Invalid version number: $opt{'compat-version'}\n" if $v >= 1000 || $s >= 1000; + $opt{'compat-version'} = sprintf "%d.%03d%03d", $r, $v, $s; +} +else { + $opt{'compat-version'} = 5; +} + +my %API = map { /^(\w+)\|([^|]*)\|([^|]*)\|(\w*)$/ + ? ( $1 => { + ($2 ? ( base => $2 ) : ()), + ($3 ? ( todo => $3 ) : ()), + (index($4, 'v') >= 0 ? ( varargs => 1 ) : ()), + (index($4, 'p') >= 0 ? ( provided => 1 ) : ()), + (index($4, 'n') >= 0 ? ( nothxarg => 1 ) : ()), + } ) + : die "invalid spec: $_" } qw( +AvFILLp|5.004050||p +AvFILL||| +CLASS|||n +CPERLscope|5.005000||p +CX_CURPAD_SAVE||| +CX_CURPAD_SV||| +CopFILEAV|5.006000||p +CopFILEGV_set|5.006000||p +CopFILEGV|5.006000||p +CopFILESV|5.006000||p +CopFILE_set|5.006000||p +CopFILE|5.006000||p +CopSTASHPV_set|5.006000||p +CopSTASHPV|5.006000||p +CopSTASH_eq|5.006000||p +CopSTASH_set|5.006000||p +CopSTASH|5.006000||p +CopyD|5.009002||p +Copy||| +CvPADLIST||| +CvSTASH||| +CvWEAKOUTSIDE||| +DEFSV_set|5.011000||p +DEFSV|5.004050||p +END_EXTERN_C|5.005000||p +ENTER||| +ERRSV|5.004050||p +EXTEND||| +EXTERN_C|5.005000||p +F0convert|||n +FREETMPS||| +GIMME_V||5.004000|n +GIMME|||n +GROK_NUMERIC_RADIX|5.007002||p +G_ARRAY||| +G_DISCARD||| +G_EVAL||| +G_METHOD|5.006001||p +G_NOARGS||| +G_SCALAR||| +G_VOID||5.004000| +GetVars||| +GvSV||| +Gv_AMupdate||| +HEf_SVKEY||5.004000| +HeHASH||5.004000| +HeKEY||5.004000| +HeKLEN||5.004000| +HePV||5.004000| +HeSVKEY_force||5.004000| +HeSVKEY_set||5.004000| +HeSVKEY||5.004000| +HeUTF8||5.011000| +HeVAL||5.004000| +HvNAME||| +INT2PTR|5.006000||p +IN_LOCALE_COMPILETIME|5.007002||p +IN_LOCALE_RUNTIME|5.007002||p +IN_LOCALE|5.007002||p +IN_PERL_COMPILETIME|5.008001||p +IS_NUMBER_GREATER_THAN_UV_MAX|5.007002||p +IS_NUMBER_INFINITY|5.007002||p +IS_NUMBER_IN_UV|5.007002||p +IS_NUMBER_NAN|5.007003||p +IS_NUMBER_NEG|5.007002||p +IS_NUMBER_NOT_INT|5.007002||p +IVSIZE|5.006000||p +IVTYPE|5.006000||p +IVdf|5.006000||p +LEAVE||| +LVRET||| +MARK||| +MULTICALL||5.011000| +MY_CXT_CLONE|5.009002||p +MY_CXT_INIT|5.007003||p +MY_CXT|5.007003||p +MoveD|5.009002||p +Move||| +NOOP|5.005000||p +NUM2PTR|5.006000||p +NVTYPE|5.006000||p +NVef|5.006001||p +NVff|5.006001||p +NVgf|5.006001||p +Newxc|5.009003||p +Newxz|5.009003||p +Newx|5.009003||p +Nullav||| +Nullch||| +Nullcv||| +Nullhv||| +Nullsv||| +ORIGMARK||| +PAD_BASE_SV||| +PAD_CLONE_VARS||| +PAD_COMPNAME_FLAGS||| +PAD_COMPNAME_GEN_set||| +PAD_COMPNAME_GEN||| +PAD_COMPNAME_OURSTASH||| +PAD_COMPNAME_PV||| +PAD_COMPNAME_TYPE||| +PAD_DUP||| +PAD_RESTORE_LOCAL||| +PAD_SAVE_LOCAL||| +PAD_SAVE_SETNULLPAD||| +PAD_SETSV||| +PAD_SET_CUR_NOSAVE||| +PAD_SET_CUR||| +PAD_SVl||| +PAD_SV||| +PERLIO_FUNCS_CAST|5.009003||p +PERLIO_FUNCS_DECL|5.009003||p +PERL_ABS|5.008001||p +PERL_BCDVERSION|5.011000||p +PERL_GCC_BRACE_GROUPS_FORBIDDEN|5.008001||p +PERL_HASH|5.004000||p +PERL_INT_MAX|5.004000||p +PERL_INT_MIN|5.004000||p +PERL_LONG_MAX|5.004000||p +PERL_LONG_MIN|5.004000||p +PERL_MAGIC_arylen|5.007002||p +PERL_MAGIC_backref|5.007002||p +PERL_MAGIC_bm|5.007002||p +PERL_MAGIC_collxfrm|5.007002||p +PERL_MAGIC_dbfile|5.007002||p +PERL_MAGIC_dbline|5.007002||p +PERL_MAGIC_defelem|5.007002||p +PERL_MAGIC_envelem|5.007002||p +PERL_MAGIC_env|5.007002||p +PERL_MAGIC_ext|5.007002||p +PERL_MAGIC_fm|5.007002||p +PERL_MAGIC_glob|5.011000||p +PERL_MAGIC_isaelem|5.007002||p +PERL_MAGIC_isa|5.007002||p +PERL_MAGIC_mutex|5.011000||p +PERL_MAGIC_nkeys|5.007002||p +PERL_MAGIC_overload_elem|5.007002||p +PERL_MAGIC_overload_table|5.007002||p +PERL_MAGIC_overload|5.007002||p +PERL_MAGIC_pos|5.007002||p +PERL_MAGIC_qr|5.007002||p +PERL_MAGIC_regdata|5.007002||p +PERL_MAGIC_regdatum|5.007002||p +PERL_MAGIC_regex_global|5.007002||p +PERL_MAGIC_shared_scalar|5.007003||p +PERL_MAGIC_shared|5.007003||p +PERL_MAGIC_sigelem|5.007002||p +PERL_MAGIC_sig|5.007002||p +PERL_MAGIC_substr|5.007002||p +PERL_MAGIC_sv|5.007002||p +PERL_MAGIC_taint|5.007002||p +PERL_MAGIC_tiedelem|5.007002||p +PERL_MAGIC_tiedscalar|5.007002||p +PERL_MAGIC_tied|5.007002||p +PERL_MAGIC_utf8|5.008001||p +PERL_MAGIC_uvar_elem|5.007003||p +PERL_MAGIC_uvar|5.007002||p +PERL_MAGIC_vec|5.007002||p +PERL_MAGIC_vstring|5.008001||p +PERL_PV_ESCAPE_ALL|5.009004||p +PERL_PV_ESCAPE_FIRSTCHAR|5.009004||p +PERL_PV_ESCAPE_NOBACKSLASH|5.009004||p +PERL_PV_ESCAPE_NOCLEAR|5.009004||p +PERL_PV_ESCAPE_QUOTE|5.009004||p +PERL_PV_ESCAPE_RE|5.009005||p +PERL_PV_ESCAPE_UNI_DETECT|5.009004||p +PERL_PV_ESCAPE_UNI|5.009004||p +PERL_PV_PRETTY_DUMP|5.009004||p +PERL_PV_PRETTY_ELLIPSES|5.010000||p +PERL_PV_PRETTY_LTGT|5.009004||p +PERL_PV_PRETTY_NOCLEAR|5.010000||p +PERL_PV_PRETTY_QUOTE|5.009004||p +PERL_PV_PRETTY_REGPROP|5.009004||p +PERL_QUAD_MAX|5.004000||p +PERL_QUAD_MIN|5.004000||p +PERL_REVISION|5.006000||p +PERL_SCAN_ALLOW_UNDERSCORES|5.007003||p +PERL_SCAN_DISALLOW_PREFIX|5.007003||p +PERL_SCAN_GREATER_THAN_UV_MAX|5.007003||p +PERL_SCAN_SILENT_ILLDIGIT|5.008001||p +PERL_SHORT_MAX|5.004000||p +PERL_SHORT_MIN|5.004000||p +PERL_SIGNALS_UNSAFE_FLAG|5.008001||p +PERL_SUBVERSION|5.006000||p +PERL_UCHAR_MAX|5.004000||p +PERL_UCHAR_MIN|5.004000||p +PERL_UINT_MAX|5.004000||p +PERL_UINT_MIN|5.004000||p +PERL_ULONG_MAX|5.004000||p +PERL_ULONG_MIN|5.004000||p +PERL_UNUSED_ARG|5.009003||p +PERL_UNUSED_CONTEXT|5.009004||p +PERL_UNUSED_DECL|5.007002||p +PERL_UNUSED_VAR|5.007002||p +PERL_UQUAD_MAX|5.004000||p +PERL_UQUAD_MIN|5.004000||p +PERL_USE_GCC_BRACE_GROUPS|5.009004||p +PERL_USHORT_MAX|5.004000||p +PERL_USHORT_MIN|5.004000||p +PERL_VERSION|5.006000||p +PL_DBsignal|5.005000||p +PL_DBsingle|||pn +PL_DBsub|||pn +PL_DBtrace|||pn +PL_Sv|5.005000||p +PL_bufend|5.011000||p +PL_bufptr|5.011000||p +PL_compiling|5.004050||p +PL_copline|5.011000||p +PL_curcop|5.004050||p +PL_curstash|5.004050||p +PL_debstash|5.004050||p +PL_defgv|5.004050||p +PL_diehook|5.004050||p +PL_dirty|5.004050||p +PL_dowarn|||pn +PL_errgv|5.004050||p +PL_expect|5.011000||p +PL_hexdigit|5.005000||p +PL_hints|5.005000||p +PL_last_in_gv|||n +PL_laststatval|5.005000||p +PL_lex_state|5.011000||p +PL_lex_stuff|5.011000||p +PL_linestr|5.011000||p +PL_modglobal||5.005000|n +PL_na|5.004050||pn +PL_no_modify|5.006000||p +PL_ofsgv|||n +PL_parser|5.009005||p +PL_perl_destruct_level|5.004050||p +PL_perldb|5.004050||p +PL_ppaddr|5.006000||p +PL_rsfp_filters|5.004050||p +PL_rsfp|5.004050||p +PL_rs|||n +PL_signals|5.008001||p +PL_stack_base|5.004050||p +PL_stack_sp|5.004050||p +PL_statcache|5.005000||p +PL_stdingv|5.004050||p +PL_sv_arenaroot|5.004050||p +PL_sv_no|5.004050||pn +PL_sv_undef|5.004050||pn +PL_sv_yes|5.004050||pn +PL_tainted|5.004050||p +PL_tainting|5.004050||p +PL_tokenbuf|5.011000||p +POP_MULTICALL||5.011000| +POPi|||n +POPl|||n +POPn|||n +POPpbytex||5.007001|n +POPpx||5.005030|n +POPp|||n +POPs|||n +PTR2IV|5.006000||p +PTR2NV|5.006000||p +PTR2UV|5.006000||p +PTR2nat|5.009003||p +PTR2ul|5.007001||p +PTRV|5.006000||p +PUSHMARK||| +PUSH_MULTICALL||5.011000| +PUSHi||| +PUSHmortal|5.009002||p +PUSHn||| +PUSHp||| +PUSHs||| +PUSHu|5.004000||p +PUTBACK||| +PerlIO_clearerr||5.007003| +PerlIO_close||5.007003| +PerlIO_context_layers||5.009004| +PerlIO_eof||5.007003| +PerlIO_error||5.007003| +PerlIO_fileno||5.007003| +PerlIO_fill||5.007003| +PerlIO_flush||5.007003| +PerlIO_get_base||5.007003| +PerlIO_get_bufsiz||5.007003| +PerlIO_get_cnt||5.007003| +PerlIO_get_ptr||5.007003| +PerlIO_read||5.007003| +PerlIO_seek||5.007003| +PerlIO_set_cnt||5.007003| +PerlIO_set_ptrcnt||5.007003| +PerlIO_setlinebuf||5.007003| +PerlIO_stderr||5.007003| +PerlIO_stdin||5.007003| +PerlIO_stdout||5.007003| +PerlIO_tell||5.007003| +PerlIO_unread||5.007003| +PerlIO_write||5.007003| +Perl_signbit||5.009005|n +PoisonFree|5.009004||p +PoisonNew|5.009004||p +PoisonWith|5.009004||p +Poison|5.008000||p +RETVAL|||n +Renewc||| +Renew||| +SAVECLEARSV||| +SAVECOMPPAD||| +SAVEPADSV||| +SAVETMPS||| +SAVE_DEFSV|5.004050||p +SPAGAIN||| +SP||| +START_EXTERN_C|5.005000||p +START_MY_CXT|5.007003||p +STMT_END|||p +STMT_START|||p +STR_WITH_LEN|5.009003||p +ST||| +SV_CONST_RETURN|5.009003||p +SV_COW_DROP_PV|5.008001||p +SV_COW_SHARED_HASH_KEYS|5.009005||p +SV_GMAGIC|5.007002||p +SV_HAS_TRAILING_NUL|5.009004||p +SV_IMMEDIATE_UNREF|5.007001||p +SV_MUTABLE_RETURN|5.009003||p +SV_NOSTEAL|5.009002||p +SV_SMAGIC|5.009003||p +SV_UTF8_NO_ENCODING|5.008001||p +SVf_UTF8|5.006000||p +SVf|5.006000||p +SVt_IV||| +SVt_NV||| +SVt_PVAV||| +SVt_PVCV||| +SVt_PVHV||| +SVt_PVMG||| +SVt_PV||| +Safefree||| +Slab_Alloc||| +Slab_Free||| +Slab_to_rw||| +StructCopy||| +SvCUR_set||| +SvCUR||| +SvEND||| +SvGAMAGIC||5.006001| +SvGETMAGIC|5.004050||p +SvGROW||| +SvIOK_UV||5.006000| +SvIOK_notUV||5.006000| +SvIOK_off||| +SvIOK_only_UV||5.006000| +SvIOK_only||| +SvIOK_on||| +SvIOKp||| +SvIOK||| +SvIVX||| +SvIV_nomg|5.009001||p +SvIV_set||| +SvIVx||| +SvIV||| +SvIsCOW_shared_hash||5.008003| +SvIsCOW||5.008003| +SvLEN_set||| +SvLEN||| +SvLOCK||5.007003| +SvMAGIC_set|5.009003||p +SvNIOK_off||| +SvNIOKp||| +SvNIOK||| +SvNOK_off||| +SvNOK_only||| +SvNOK_on||| +SvNOKp||| +SvNOK||| +SvNVX||| +SvNV_set||| +SvNVx||| +SvNV||| +SvOK||| +SvOOK_offset||5.011000| +SvOOK||| +SvPOK_off||| +SvPOK_only_UTF8||5.006000| +SvPOK_only||| +SvPOK_on||| +SvPOKp||| +SvPOK||| +SvPVX_const|5.009003||p +SvPVX_mutable|5.009003||p +SvPVX||| +SvPV_const|5.009003||p +SvPV_flags_const_nolen|5.009003||p +SvPV_flags_const|5.009003||p +SvPV_flags_mutable|5.009003||p +SvPV_flags|5.007002||p +SvPV_force_flags_mutable|5.009003||p +SvPV_force_flags_nolen|5.009003||p +SvPV_force_flags|5.007002||p +SvPV_force_mutable|5.009003||p +SvPV_force_nolen|5.009003||p +SvPV_force_nomg_nolen|5.009003||p +SvPV_force_nomg|5.007002||p +SvPV_force|||p +SvPV_mutable|5.009003||p +SvPV_nolen_const|5.009003||p +SvPV_nolen|5.006000||p +SvPV_nomg_const_nolen|5.009003||p +SvPV_nomg_const|5.009003||p +SvPV_nomg|5.007002||p +SvPV_renew|5.009003||p +SvPV_set||| +SvPVbyte_force||5.009002| +SvPVbyte_nolen||5.006000| +SvPVbytex_force||5.006000| +SvPVbytex||5.006000| +SvPVbyte|5.006000||p +SvPVutf8_force||5.006000| +SvPVutf8_nolen||5.006000| +SvPVutf8x_force||5.006000| +SvPVutf8x||5.006000| +SvPVutf8||5.006000| +SvPVx||| +SvPV||| +SvREFCNT_dec||| +SvREFCNT_inc_NN|5.009004||p +SvREFCNT_inc_simple_NN|5.009004||p +SvREFCNT_inc_simple_void_NN|5.009004||p +SvREFCNT_inc_simple_void|5.009004||p +SvREFCNT_inc_simple|5.009004||p +SvREFCNT_inc_void_NN|5.009004||p +SvREFCNT_inc_void|5.009004||p +SvREFCNT_inc|||p +SvREFCNT||| +SvROK_off||| +SvROK_on||| +SvROK||| +SvRV_set|5.009003||p +SvRV||| +SvRXOK||5.009005| +SvRX||5.009005| +SvSETMAGIC||| +SvSHARED_HASH|5.009003||p +SvSHARE||5.007003| +SvSTASH_set|5.009003||p +SvSTASH||| +SvSetMagicSV_nosteal||5.004000| +SvSetMagicSV||5.004000| +SvSetSV_nosteal||5.004000| +SvSetSV||| +SvTAINTED_off||5.004000| +SvTAINTED_on||5.004000| +SvTAINTED||5.004000| +SvTAINT||| +SvTRUE||| +SvTYPE||| +SvUNLOCK||5.007003| +SvUOK|5.007001|5.006000|p +SvUPGRADE||| +SvUTF8_off||5.006000| +SvUTF8_on||5.006000| +SvUTF8||5.006000| +SvUVXx|5.004000||p +SvUVX|5.004000||p +SvUV_nomg|5.009001||p +SvUV_set|5.009003||p +SvUVx|5.004000||p +SvUV|5.004000||p +SvVOK||5.008001| +SvVSTRING_mg|5.009004||p +THIS|||n +UNDERBAR|5.009002||p +UTF8_MAXBYTES|5.009002||p +UVSIZE|5.006000||p +UVTYPE|5.006000||p +UVXf|5.007001||p +UVof|5.006000||p +UVuf|5.006000||p +UVxf|5.006000||p +WARN_ALL|5.006000||p +WARN_AMBIGUOUS|5.006000||p +WARN_ASSERTIONS|5.011000||p +WARN_BAREWORD|5.006000||p +WARN_CLOSED|5.006000||p +WARN_CLOSURE|5.006000||p +WARN_DEBUGGING|5.006000||p +WARN_DEPRECATED|5.006000||p +WARN_DIGIT|5.006000||p +WARN_EXEC|5.006000||p +WARN_EXITING|5.006000||p +WARN_GLOB|5.006000||p +WARN_INPLACE|5.006000||p +WARN_INTERNAL|5.006000||p +WARN_IO|5.006000||p +WARN_LAYER|5.008000||p +WARN_MALLOC|5.006000||p +WARN_MISC|5.006000||p +WARN_NEWLINE|5.006000||p +WARN_NUMERIC|5.006000||p +WARN_ONCE|5.006000||p +WARN_OVERFLOW|5.006000||p +WARN_PACK|5.006000||p +WARN_PARENTHESIS|5.006000||p +WARN_PIPE|5.006000||p +WARN_PORTABLE|5.006000||p +WARN_PRECEDENCE|5.006000||p +WARN_PRINTF|5.006000||p +WARN_PROTOTYPE|5.006000||p +WARN_QW|5.006000||p +WARN_RECURSION|5.006000||p +WARN_REDEFINE|5.006000||p +WARN_REGEXP|5.006000||p +WARN_RESERVED|5.006000||p +WARN_SEMICOLON|5.006000||p +WARN_SEVERE|5.006000||p +WARN_SIGNAL|5.006000||p +WARN_SUBSTR|5.006000||p +WARN_SYNTAX|5.006000||p +WARN_TAINT|5.006000||p +WARN_THREADS|5.008000||p +WARN_UNINITIALIZED|5.006000||p +WARN_UNOPENED|5.006000||p +WARN_UNPACK|5.006000||p +WARN_UNTIE|5.006000||p +WARN_UTF8|5.006000||p +WARN_VOID|5.006000||p +XCPT_CATCH|5.009002||p +XCPT_RETHROW|5.009002||p +XCPT_TRY_END|5.009002||p +XCPT_TRY_START|5.009002||p +XPUSHi||| +XPUSHmortal|5.009002||p +XPUSHn||| +XPUSHp||| +XPUSHs||| +XPUSHu|5.004000||p +XSRETURN_EMPTY||| +XSRETURN_IV||| +XSRETURN_NO||| +XSRETURN_NV||| +XSRETURN_PV||| +XSRETURN_UNDEF||| +XSRETURN_UV|5.008001||p +XSRETURN_YES||| +XSRETURN|||p +XST_mIV||| +XST_mNO||| +XST_mNV||| +XST_mPV||| +XST_mUNDEF||| +XST_mUV|5.008001||p +XST_mYES||| +XS_VERSION_BOOTCHECK||| +XS_VERSION||| +XSprePUSH|5.006000||p +XS||| +ZeroD|5.009002||p +Zero||| +_aMY_CXT|5.007003||p +_pMY_CXT|5.007003||p +aMY_CXT_|5.007003||p +aMY_CXT|5.007003||p +aTHXR_|5.011000||p +aTHXR|5.011000||p +aTHX_|5.006000||p +aTHX|5.006000||p +add_data|||n +addmad||| +allocmy||| +amagic_call||| +amagic_cmp_locale||| +amagic_cmp||| +amagic_i_ncmp||| +amagic_ncmp||| +any_dup||| +ao||| +append_elem||| +append_list||| +append_madprops||| +apply_attrs_my||| +apply_attrs_string||5.006001| +apply_attrs||| +apply||| +atfork_lock||5.007003|n +atfork_unlock||5.007003|n +av_arylen_p||5.009003| +av_clear||| +av_create_and_push||5.009005| +av_create_and_unshift_one||5.009005| +av_delete||5.006000| +av_exists||5.006000| +av_extend||| +av_fetch||| +av_fill||| +av_iter_p||5.011000| +av_len||| +av_make||| +av_pop||| +av_push||| +av_reify||| +av_shift||| +av_store||| +av_undef||| +av_unshift||| +ax|||n +bad_type||| +bind_match||| +block_end||| +block_gimme||5.004000| +block_start||| +boolSV|5.004000||p +boot_core_PerlIO||| +boot_core_UNIVERSAL||| +boot_core_mro||| +boot_core_xsutils||| +bytes_from_utf8||5.007001| +bytes_to_uni|||n +bytes_to_utf8||5.006001| +call_argv|5.006000||p +call_atexit||5.006000| +call_list||5.004000| +call_method|5.006000||p +call_pv|5.006000||p +call_sv|5.006000||p +calloc||5.007002|n +cando||| +cast_i32||5.006000| +cast_iv||5.006000| +cast_ulong||5.006000| +cast_uv||5.006000| +check_type_and_open||| +check_uni||| +checkcomma||| +checkposixcc||| +ckWARN|5.006000||p +ck_anoncode||| +ck_bitop||| +ck_concat||| +ck_defined||| +ck_delete||| +ck_die||| +ck_each||| +ck_eof||| +ck_eval||| +ck_exec||| +ck_exists||| +ck_exit||| +ck_ftst||| +ck_fun||| +ck_glob||| +ck_grep||| +ck_index||| +ck_join||| +ck_lfun||| +ck_listiob||| +ck_match||| +ck_method||| +ck_null||| +ck_open||| +ck_readline||| +ck_repeat||| +ck_require||| +ck_return||| +ck_rfun||| +ck_rvconst||| +ck_sassign||| +ck_select||| +ck_shift||| +ck_sort||| +ck_spair||| +ck_split||| +ck_subr||| +ck_substr||| +ck_svconst||| +ck_trunc||| +ck_unpack||| +ckwarn_d||5.009003| +ckwarn||5.009003| +cl_and|||n +cl_anything|||n +cl_init_zero|||n +cl_init|||n +cl_is_anything|||n +cl_or|||n +clear_placeholders||| +closest_cop||| +convert||| +cop_free||| +cr_textfilter||| +create_eval_scope||| +croak_nocontext|||vn +croak_xs_usage||5.011000| +croak|||v +csighandler||5.009003|n +curmad||| +custom_op_desc||5.007003| +custom_op_name||5.007003| +cv_ckproto_len||| +cv_clone||| +cv_const_sv||5.004000| +cv_dump||| +cv_undef||| +cx_dump||5.005000| +cx_dup||| +cxinc||| +dAXMARK|5.009003||p +dAX|5.007002||p +dITEMS|5.007002||p +dMARK||| +dMULTICALL||5.009003| +dMY_CXT_SV|5.007003||p +dMY_CXT|5.007003||p +dNOOP|5.006000||p +dORIGMARK||| +dSP||| +dTHR|5.004050||p +dTHXR|5.011000||p +dTHXa|5.006000||p +dTHXoa|5.006000||p +dTHX|5.006000||p +dUNDERBAR|5.009002||p +dVAR|5.009003||p +dXCPT|5.009002||p +dXSARGS||| +dXSI32||| +dXSTARG|5.006000||p +deb_curcv||| +deb_nocontext|||vn +deb_stack_all||| +deb_stack_n||| +debop||5.005000| +debprofdump||5.005000| +debprof||| +debstackptrs||5.007003| +debstack||5.007003| +debug_start_match||| +deb||5.007003|v +del_sv||| +delete_eval_scope||| +delimcpy||5.004000| +deprecate_old||| +deprecate||| +despatch_signals||5.007001| +destroy_matcher||| +die_nocontext|||vn +die_where||| +die|||v +dirp_dup||| +div128||| +djSP||| +do_aexec5||| +do_aexec||| +do_aspawn||| +do_binmode||5.004050| +do_chomp||| +do_chop||| +do_close||| +do_dump_pad||| +do_eof||| +do_exec3||| +do_execfree||| +do_exec||| +do_gv_dump||5.006000| +do_gvgv_dump||5.006000| +do_hv_dump||5.006000| +do_ipcctl||| +do_ipcget||| +do_join||| +do_kv||| +do_magic_dump||5.006000| +do_msgrcv||| +do_msgsnd||| +do_oddball||| +do_op_dump||5.006000| +do_op_xmldump||| +do_open9||5.006000| +do_openn||5.007001| +do_open||5.004000| +do_pmop_dump||5.006000| +do_pmop_xmldump||| +do_print||| +do_readline||| +do_seek||| +do_semop||| +do_shmio||| +do_smartmatch||| +do_spawn_nowait||| +do_spawn||| +do_sprintf||| +do_sv_dump||5.006000| +do_sysseek||| +do_tell||| +do_trans_complex_utf8||| +do_trans_complex||| +do_trans_count_utf8||| +do_trans_count||| +do_trans_simple_utf8||| +do_trans_simple||| +do_trans||| +do_vecget||| +do_vecset||| +do_vop||| +docatch||| +doeval||| +dofile||| +dofindlabel||| +doform||| +doing_taint||5.008001|n +dooneliner||| +doopen_pm||| +doparseform||| +dopoptoeval||| +dopoptogiven||| +dopoptolabel||| +dopoptoloop||| +dopoptosub_at||| +dopoptowhen||| +doref||5.009003| +dounwind||| +dowantarray||| +dump_all||5.006000| +dump_eval||5.006000| +dump_exec_pos||| +dump_fds||| +dump_form||5.006000| +dump_indent||5.006000|v +dump_mstats||| +dump_packsubs||5.006000| +dump_sub||5.006000| +dump_sv_child||| +dump_trie_interim_list||| +dump_trie_interim_table||| +dump_trie||| +dump_vindent||5.006000| +dumpuntil||| +dup_attrlist||| +emulate_cop_io||| +eval_pv|5.006000||p +eval_sv|5.006000||p +exec_failed||| +expect_number||| +fbm_compile||5.005000| +fbm_instr||5.005000| +feature_is_enabled||| +fetch_cop_label||5.011000| +filter_add||| +filter_del||| +filter_gets||| +filter_read||| +find_and_forget_pmops||| +find_array_subscript||| +find_beginning||| +find_byclass||| +find_hash_subscript||| +find_in_my_stash||| +find_runcv||5.008001| +find_rundefsvoffset||5.009002| +find_script||| +find_uninit_var||| +first_symbol|||n +fold_constants||| +forbid_setid||| +force_ident||| +force_list||| +force_next||| +force_version||| +force_word||| +forget_pmop||| +form_nocontext|||vn +form||5.004000|v +fp_dup||| +fprintf_nocontext|||vn +free_global_struct||| +free_tied_hv_pool||| +free_tmps||| +gen_constant_list||| +get_arena||| +get_aux_mg||| +get_av|5.006000||p +get_context||5.006000|n +get_cvn_flags||5.009005| +get_cv|5.006000||p +get_db_sub||| +get_debug_opts||| +get_hash_seed||| +get_hv|5.006000||p +get_isa_hash||| +get_mstats||| +get_no_modify||| +get_num||| +get_op_descs||5.005000| +get_op_names||5.005000| +get_opargs||| +get_ppaddr||5.006000| +get_re_arg||| +get_sv|5.006000||p +get_vtbl||5.005030| +getcwd_sv||5.007002| +getenv_len||| +glob_2number||| +glob_2pv||| +glob_assign_glob||| +glob_assign_ref||| +gp_dup||| +gp_free||| +gp_ref||| +grok_bin|5.007003||p +grok_hex|5.007003||p +grok_number|5.007002||p +grok_numeric_radix|5.007002||p +grok_oct|5.007003||p +group_end||| +gv_AVadd||| +gv_HVadd||| +gv_IOadd||| +gv_SVadd||| +gv_autoload4||5.004000| +gv_check||| +gv_const_sv||5.009003| +gv_dump||5.006000| +gv_efullname3||5.004000| +gv_efullname4||5.006001| +gv_efullname||| +gv_ename||| +gv_fetchfile_flags||5.009005| +gv_fetchfile||| +gv_fetchmeth_autoload||5.007003| +gv_fetchmethod_autoload||5.004000| +gv_fetchmethod_flags||5.011000| +gv_fetchmethod||| +gv_fetchmeth||| +gv_fetchpvn_flags||5.009002| +gv_fetchpv||| +gv_fetchsv||5.009002| +gv_fullname3||5.004000| +gv_fullname4||5.006001| +gv_fullname||| +gv_get_super_pkg||| +gv_handler||5.007001| +gv_init_sv||| +gv_init||| +gv_name_set||5.009004| +gv_stashpvn|5.004000||p +gv_stashpvs||5.009003| +gv_stashpv||| +gv_stashsv||| +he_dup||| +hek_dup||| +hfreeentries||| +hsplit||| +hv_assert||5.011000| +hv_auxinit|||n +hv_backreferences_p||| +hv_clear_placeholders||5.009001| +hv_clear||| +hv_common_key_len||5.010000| +hv_common||5.010000| +hv_copy_hints_hv||| +hv_delayfree_ent||5.004000| +hv_delete_common||| +hv_delete_ent||5.004000| +hv_delete||| +hv_eiter_p||5.009003| +hv_eiter_set||5.009003| +hv_exists_ent||5.004000| +hv_exists||| +hv_fetch_ent||5.004000| +hv_fetchs|5.009003||p +hv_fetch||| +hv_free_ent||5.004000| +hv_iterinit||| +hv_iterkeysv||5.004000| +hv_iterkey||| +hv_iternext_flags||5.008000| +hv_iternextsv||| +hv_iternext||| +hv_iterval||| +hv_kill_backrefs||| +hv_ksplit||5.004000| +hv_magic_check|||n +hv_magic||| +hv_name_set||5.009003| +hv_notallowed||| +hv_placeholders_get||5.009003| +hv_placeholders_p||5.009003| +hv_placeholders_set||5.009003| +hv_riter_p||5.009003| +hv_riter_set||5.009003| +hv_scalar||5.009001| +hv_store_ent||5.004000| +hv_store_flags||5.008000| +hv_stores|5.009004||p +hv_store||| +hv_undef||| +ibcmp_locale||5.004000| +ibcmp_utf8||5.007003| +ibcmp||| +incline||| +incpush_if_exists||| +incpush_use_sep||| +incpush||| +ingroup||| +init_argv_symbols||| +init_debugger||| +init_global_struct||| +init_i18nl10n||5.006000| +init_i18nl14n||5.006000| +init_ids||| +init_interp||| +init_main_stash||| +init_perllib||| +init_postdump_symbols||| +init_predump_symbols||| +init_stacks||5.005000| +init_tm||5.007002| +instr||| +intro_my||| +intuit_method||| +intuit_more||| +invert||| +io_close||| +isALNUMC|5.006000||p +isALNUM||| +isALPHA||| +isASCII|5.006000||p +isBLANK|5.006001||p +isCNTRL|5.006000||p +isDIGIT||| +isGRAPH|5.006000||p +isLOWER||| +isPRINT|5.004000||p +isPSXSPC|5.006001||p +isPUNCT|5.006000||p +isSPACE||| +isUPPER||| +isXDIGIT|5.006000||p +is_an_int||| +is_gv_magical_sv||| +is_handle_constructor|||n +is_list_assignment||| +is_lvalue_sub||5.007001| +is_uni_alnum_lc||5.006000| +is_uni_alnumc_lc||5.006000| +is_uni_alnumc||5.006000| +is_uni_alnum||5.006000| +is_uni_alpha_lc||5.006000| +is_uni_alpha||5.006000| +is_uni_ascii_lc||5.006000| +is_uni_ascii||5.006000| +is_uni_cntrl_lc||5.006000| +is_uni_cntrl||5.006000| +is_uni_digit_lc||5.006000| +is_uni_digit||5.006000| +is_uni_graph_lc||5.006000| +is_uni_graph||5.006000| +is_uni_idfirst_lc||5.006000| +is_uni_idfirst||5.006000| +is_uni_lower_lc||5.006000| +is_uni_lower||5.006000| +is_uni_print_lc||5.006000| +is_uni_print||5.006000| +is_uni_punct_lc||5.006000| +is_uni_punct||5.006000| +is_uni_space_lc||5.006000| +is_uni_space||5.006000| +is_uni_upper_lc||5.006000| +is_uni_upper||5.006000| +is_uni_xdigit_lc||5.006000| +is_uni_xdigit||5.006000| +is_utf8_alnumc||5.006000| +is_utf8_alnum||5.006000| +is_utf8_alpha||5.006000| +is_utf8_ascii||5.006000| +is_utf8_char_slow|||n +is_utf8_char||5.006000| +is_utf8_cntrl||5.006000| +is_utf8_common||| +is_utf8_digit||5.006000| +is_utf8_graph||5.006000| +is_utf8_idcont||5.008000| +is_utf8_idfirst||5.006000| +is_utf8_lower||5.006000| +is_utf8_mark||5.006000| +is_utf8_print||5.006000| +is_utf8_punct||5.006000| +is_utf8_space||5.006000| +is_utf8_string_loclen||5.009003| +is_utf8_string_loc||5.008001| +is_utf8_string||5.006001| +is_utf8_upper||5.006000| +is_utf8_xdigit||5.006000| +isa_lookup||| +items|||n +ix|||n +jmaybe||| +join_exact||| +keyword||| +leave_scope||| +lex_end||| +lex_start||| +linklist||| +listkids||| +list||| +load_module_nocontext|||vn +load_module|5.006000||pv +localize||| +looks_like_bool||| +looks_like_number||| +lop||| +mPUSHi|5.009002||p +mPUSHn|5.009002||p +mPUSHp|5.009002||p +mPUSHs|5.011000||p +mPUSHu|5.009002||p +mXPUSHi|5.009002||p +mXPUSHn|5.009002||p +mXPUSHp|5.009002||p +mXPUSHs|5.011000||p +mXPUSHu|5.009002||p +mad_free||| +madlex||| +madparse||| +magic_clear_all_env||| +magic_clearenv||| +magic_clearhint||| +magic_clearisa||| +magic_clearpack||| +magic_clearsig||| +magic_dump||5.006000| +magic_existspack||| +magic_freearylen_p||| +magic_freeovrld||| +magic_getarylen||| +magic_getdefelem||| +magic_getnkeys||| +magic_getpack||| +magic_getpos||| +magic_getsig||| +magic_getsubstr||| +magic_gettaint||| +magic_getuvar||| +magic_getvec||| +magic_get||| +magic_killbackrefs||| +magic_len||| +magic_methcall||| +magic_methpack||| +magic_nextpack||| +magic_regdata_cnt||| +magic_regdatum_get||| +magic_regdatum_set||| +magic_scalarpack||| +magic_set_all_env||| +magic_setamagic||| +magic_setarylen||| +magic_setcollxfrm||| +magic_setdbline||| +magic_setdefelem||| +magic_setenv||| +magic_sethint||| +magic_setisa||| +magic_setmglob||| +magic_setnkeys||| +magic_setpack||| +magic_setpos||| +magic_setregexp||| +magic_setsig||| +magic_setsubstr||| +magic_settaint||| +magic_setutf8||| +magic_setuvar||| +magic_setvec||| +magic_set||| +magic_sizepack||| +magic_wipepack||| +make_matcher||| +make_trie_failtable||| +make_trie||| +malloc_good_size|||n +malloced_size|||n +malloc||5.007002|n +markstack_grow||| +matcher_matches_sv||| +measure_struct||| +memEQ|5.004000||p +memNE|5.004000||p +mem_collxfrm||| +mem_log_common|||n +mess_alloc||| +mess_nocontext|||vn +mess||5.006000|v +method_common||| +mfree||5.007002|n +mg_clear||| +mg_copy||| +mg_dup||| +mg_find||| +mg_free||| +mg_get||| +mg_length||5.005000| +mg_localize||| +mg_magical||| +mg_set||| +mg_size||5.005000| +mini_mktime||5.007002| +missingterm||| +mode_from_discipline||| +modkids||| +mod||| +more_bodies||| +more_sv||| +moreswitches||| +mro_get_from_name||5.011000| +mro_get_linear_isa_dfs||| +mro_get_linear_isa||5.009005| +mro_get_private_data||5.011000| +mro_isa_changed_in||| +mro_meta_dup||| +mro_meta_init||| +mro_method_changed_in||5.009005| +mro_register||5.011000| +mro_set_mro||5.011000| +mro_set_private_data||5.011000| +mul128||| +mulexp10|||n +my_atof2||5.007002| +my_atof||5.006000| +my_attrs||| +my_bcopy|||n +my_betoh16|||n +my_betoh32|||n +my_betoh64|||n +my_betohi|||n +my_betohl|||n +my_betohs|||n +my_bzero|||n +my_chsize||| +my_clearenv||| +my_cxt_index||| +my_cxt_init||| +my_dirfd||5.009005| +my_exit_jump||| +my_exit||| +my_failure_exit||5.004000| +my_fflush_all||5.006000| +my_fork||5.007003|n +my_htobe16|||n +my_htobe32|||n +my_htobe64|||n +my_htobei|||n +my_htobel|||n +my_htobes|||n +my_htole16|||n +my_htole32|||n +my_htole64|||n +my_htolei|||n +my_htolel|||n +my_htoles|||n +my_htonl||| +my_kid||| +my_letoh16|||n +my_letoh32|||n +my_letoh64|||n +my_letohi|||n +my_letohl|||n +my_letohs|||n +my_lstat||| +my_memcmp||5.004000|n +my_memset|||n +my_ntohl||| +my_pclose||5.004000| +my_popen_list||5.007001| +my_popen||5.004000| +my_setenv||| +my_snprintf|5.009004||pvn +my_socketpair||5.007003|n +my_sprintf|5.009003||pvn +my_stat||| +my_strftime||5.007002| +my_strlcat|5.009004||pn +my_strlcpy|5.009004||pn +my_swabn|||n +my_swap||| +my_unexec||| +my_vsnprintf||5.009004|n +need_utf8|||n +newANONATTRSUB||5.006000| +newANONHASH||| +newANONLIST||| +newANONSUB||| +newASSIGNOP||| +newATTRSUB||5.006000| +newAVREF||| +newAV||| +newBINOP||| +newCONDOP||| +newCONSTSUB|5.004050||p +newCVREF||| +newDEFSVOP||| +newFORM||| +newFOROP||| +newGIVENOP||5.009003| +newGIVWHENOP||| +newGP||| +newGVOP||| +newGVREF||| +newGVgen||| +newHVREF||| +newHVhv||5.005000| +newHV||| +newIO||| +newLISTOP||| +newLOGOP||| +newLOOPEX||| +newLOOPOP||| +newMADPROP||| +newMADsv||| +newMYSUB||| +newNULLLIST||| +newOP||| +newPADOP||| +newPMOP||| +newPROG||| +newPVOP||| +newRANGE||| +newRV_inc|5.004000||p +newRV_noinc|5.004000||p +newRV||| +newSLICEOP||| +newSTATEOP||| +newSUB||| +newSVOP||| +newSVREF||| +newSV_type||5.009005| +newSVhek||5.009003| +newSViv||| +newSVnv||| +newSVpvf_nocontext|||vn +newSVpvf||5.004000|v +newSVpvn_flags|5.011000||p +newSVpvn_share|5.007001||p +newSVpvn_utf8|5.011000||p +newSVpvn|5.004050||p +newSVpvs_flags|5.011000||p +newSVpvs_share||5.009003| +newSVpvs|5.009003||p +newSVpv||| +newSVrv||| +newSVsv||| +newSVuv|5.006000||p +newSV||| +newTOKEN||| +newUNOP||| +newWHENOP||5.009003| +newWHILEOP||5.009003| +newXS_flags||5.009004| +newXSproto||5.006000| +newXS||5.006000| +new_collate||5.006000| +new_constant||| +new_ctype||5.006000| +new_he||| +new_logop||| +new_numeric||5.006000| +new_stackinfo||5.005000| +new_version||5.009000| +new_warnings_bitfield||| +next_symbol||| +nextargv||| +nextchar||| +ninstr||| +no_bareword_allowed||| +no_fh_allowed||| +no_op||| +not_a_number||| +nothreadhook||5.008000| +nuke_stacks||| +num_overflow|||n +offer_nice_chunk||| +oopsAV||| +oopsHV||| +op_clear||| +op_const_sv||| +op_dump||5.006000| +op_free||| +op_getmad_weak||| +op_getmad||| +op_null||5.007002| +op_refcnt_dec||| +op_refcnt_inc||| +op_refcnt_lock||5.009002| +op_refcnt_unlock||5.009002| +op_xmldump||| +open_script||| +pMY_CXT_|5.007003||p +pMY_CXT|5.007003||p +pTHX_|5.006000||p +pTHX|5.006000||p +packWARN|5.007003||p +pack_cat||5.007003| +pack_rec||| +package||| +packlist||5.008001| +pad_add_anon||| +pad_add_name||| +pad_alloc||| +pad_block_start||| +pad_check_dup||| +pad_compname_type||| +pad_findlex||| +pad_findmy||| +pad_fixup_inner_anons||| +pad_free||| +pad_leavemy||| +pad_new||| +pad_peg|||n +pad_push||| +pad_reset||| +pad_setsv||| +pad_sv||5.011000| +pad_swipe||| +pad_tidy||| +pad_undef||| +parse_body||| +parse_unicode_opts||| +parser_dup||| +parser_free||| +path_is_absolute|||n +peep||| +pending_Slabs_to_ro||| +perl_alloc_using|||n +perl_alloc|||n +perl_clone_using|||n +perl_clone|||n +perl_construct|||n +perl_destruct||5.007003|n +perl_free|||n +perl_parse||5.006000|n +perl_run|||n +pidgone||| +pm_description||| +pmflag||| +pmop_dump||5.006000| +pmop_xmldump||| +pmruntime||| +pmtrans||| +pop_scope||| +pregcomp||5.009005| +pregexec||| +pregfree2||5.011000| +pregfree||| +prepend_elem||| +prepend_madprops||| +printbuf||| +printf_nocontext|||vn +process_special_blocks||| +ptr_table_clear||5.009005| +ptr_table_fetch||5.009005| +ptr_table_find|||n +ptr_table_free||5.009005| +ptr_table_new||5.009005| +ptr_table_split||5.009005| +ptr_table_store||5.009005| +push_scope||| +put_byte||| +pv_display|5.006000||p +pv_escape|5.009004||p +pv_pretty|5.009004||p +pv_uni_display||5.007003| +qerror||| +qsortsvu||| +re_compile||5.009005| +re_croak2||| +re_dup_guts||| +re_intuit_start||5.009005| +re_intuit_string||5.006000| +readpipe_override||| +realloc||5.007002|n +reentrant_free||| +reentrant_init||| +reentrant_retry|||vn +reentrant_size||| +ref_array_or_hash||| +refcounted_he_chain_2hv||| +refcounted_he_fetch||| +refcounted_he_free||| +refcounted_he_new_common||| +refcounted_he_new||| +refcounted_he_value||| +refkids||| +refto||| +ref||5.011000| +reg_check_named_buff_matched||| +reg_named_buff_all||5.009005| +reg_named_buff_exists||5.009005| +reg_named_buff_fetch||5.009005| +reg_named_buff_firstkey||5.009005| +reg_named_buff_iter||| +reg_named_buff_nextkey||5.009005| +reg_named_buff_scalar||5.009005| +reg_named_buff||| +reg_namedseq||| +reg_node||| +reg_numbered_buff_fetch||| +reg_numbered_buff_length||| +reg_numbered_buff_store||| +reg_qr_package||| +reg_recode||| +reg_scan_name||| +reg_skipcomment||| +reg_temp_copy||| +reganode||| +regatom||| +regbranch||| +regclass_swash||5.009004| +regclass||| +regcppop||| +regcppush||| +regcurly|||n +regdump_extflags||| +regdump||5.005000| +regdupe_internal||| +regexec_flags||5.005000| +regfree_internal||5.009005| +reghop3|||n +reghop4|||n +reghopmaybe3|||n +reginclass||| +reginitcolors||5.006000| +reginsert||| +regmatch||| +regnext||5.005000| +regpiece||| +regpposixcc||| +regprop||| +regrepeat||| +regtail_study||| +regtail||| +regtry||| +reguni||| +regwhite|||n +reg||| +repeatcpy||| +report_evil_fh||| +report_uninit||| +require_pv||5.006000| +require_tie_mod||| +restore_magic||| +rninstr||| +rsignal_restore||| +rsignal_save||| +rsignal_state||5.004000| +rsignal||5.004000| +run_body||| +run_user_filter||| +runops_debug||5.005000| +runops_standard||5.005000| +rvpv_dup||| +rxres_free||| +rxres_restore||| +rxres_save||| +safesyscalloc||5.006000|n +safesysfree||5.006000|n +safesysmalloc||5.006000|n +safesysrealloc||5.006000|n +same_dirent||| +save_I16||5.004000| +save_I32||| +save_I8||5.006000| +save_adelete||5.011000| +save_aelem||5.004050| +save_alloc||5.006000| +save_aptr||| +save_ary||| +save_bool||5.008001| +save_clearsv||| +save_delete||| +save_destructor_x||5.006000| +save_destructor||5.006000| +save_freeop||| +save_freepv||| +save_freesv||| +save_generic_pvref||5.006001| +save_generic_svref||5.005030| +save_gp||5.004000| +save_hash||| +save_hek_flags|||n +save_helem_flags||5.011000| +save_helem||5.004050| +save_hints||| +save_hptr||| +save_int||| +save_item||| +save_iv||5.005000| +save_lines||| +save_list||| +save_long||| +save_magic||| +save_mortalizesv||5.007001| +save_nogv||| +save_op||| +save_padsv_and_mortalize||5.011000| +save_pptr||| +save_pushi32ptr||| +save_pushptri32ptr||| +save_pushptrptr||| +save_pushptr||5.011000| +save_re_context||5.006000| +save_scalar_at||| +save_scalar||| +save_set_svflags||5.009000| +save_shared_pvref||5.007003| +save_sptr||| +save_svref||| +save_vptr||5.006000| +savepvn||| +savepvs||5.009003| +savepv||| +savesharedpvn||5.009005| +savesharedpv||5.007003| +savestack_grow_cnt||5.008001| +savestack_grow||| +savesvpv||5.009002| +sawparens||| +scalar_mod_type|||n +scalarboolean||| +scalarkids||| +scalarseq||| +scalarvoid||| +scalar||| +scan_bin||5.006000| +scan_commit||| +scan_const||| +scan_formline||| +scan_heredoc||| +scan_hex||| +scan_ident||| +scan_inputsymbol||| +scan_num||5.007001| +scan_oct||| +scan_pat||| +scan_str||| +scan_subst||| +scan_trans||| +scan_version||5.009001| +scan_vstring||5.009005| +scan_word||| +scope||| +screaminstr||5.005000| +search_const||| +seed||5.008001| +sequence_num||| +sequence_tail||| +sequence||| +set_context||5.006000|n +set_numeric_local||5.006000| +set_numeric_radix||5.006000| +set_numeric_standard||5.006000| +setdefout||| +share_hek_flags||| +share_hek||5.004000| +si_dup||| +sighandler|||n +simplify_sort||| +skipspace0||| +skipspace1||| +skipspace2||| +skipspace||| +softref2xv||| +sortcv_stacked||| +sortcv_xsub||| +sortcv||| +sortsv_flags||5.009003| +sortsv||5.007003| +space_join_names_mortal||| +ss_dup||| +stack_grow||| +start_force||| +start_glob||| +start_subparse||5.004000| +stashpv_hvname_match||5.011000| +stdize_locale||| +store_cop_label||| +strEQ||| +strGE||| +strGT||| +strLE||| +strLT||| +strNE||| +str_to_version||5.006000| +strip_return||| +strnEQ||| +strnNE||| +study_chunk||| +sub_crush_depth||| +sublex_done||| +sublex_push||| +sublex_start||| +sv_2bool||| +sv_2cv||| +sv_2io||| +sv_2iuv_common||| +sv_2iuv_non_preserve||| +sv_2iv_flags||5.009001| +sv_2iv||| +sv_2mortal||| +sv_2num||| +sv_2nv||| +sv_2pv_flags|5.007002||p +sv_2pv_nolen|5.006000||p +sv_2pvbyte_nolen|5.006000||p +sv_2pvbyte|5.006000||p +sv_2pvutf8_nolen||5.006000| +sv_2pvutf8||5.006000| +sv_2pv||| +sv_2uv_flags||5.009001| +sv_2uv|5.004000||p +sv_add_arena||| +sv_add_backref||| +sv_backoff||| +sv_bless||| +sv_cat_decode||5.008001| +sv_catpv_mg|5.004050||p +sv_catpvf_mg_nocontext|||pvn +sv_catpvf_mg|5.006000|5.004000|pv +sv_catpvf_nocontext|||vn +sv_catpvf||5.004000|v +sv_catpvn_flags||5.007002| +sv_catpvn_mg|5.004050||p +sv_catpvn_nomg|5.007002||p +sv_catpvn||| +sv_catpvs|5.009003||p +sv_catpv||| +sv_catsv_flags||5.007002| +sv_catsv_mg|5.004050||p +sv_catsv_nomg|5.007002||p +sv_catsv||| +sv_catxmlpvn||| +sv_catxmlsv||| +sv_chop||| +sv_clean_all||| +sv_clean_objs||| +sv_clear||| +sv_cmp_locale||5.004000| +sv_cmp||| +sv_collxfrm||| +sv_compile_2op||5.008001| +sv_copypv||5.007003| +sv_dec||| +sv_del_backref||| +sv_derived_from||5.004000| +sv_destroyable||5.010000| +sv_does||5.009004| +sv_dump||| +sv_dup||| +sv_eq||| +sv_exp_grow||| +sv_force_normal_flags||5.007001| +sv_force_normal||5.006000| +sv_free2||| +sv_free_arenas||| +sv_free||| +sv_gets||5.004000| +sv_grow||| +sv_i_ncmp||| +sv_inc||| +sv_insert_flags||5.011000| +sv_insert||| +sv_isa||| +sv_isobject||| +sv_iv||5.005000| +sv_kill_backrefs||| +sv_len_utf8||5.006000| +sv_len||| +sv_magic_portable|5.011000|5.004000|p +sv_magicext||5.007003| +sv_magic||| +sv_mortalcopy||| +sv_ncmp||| +sv_newmortal||| +sv_newref||| +sv_nolocking||5.007003| +sv_nosharing||5.007003| +sv_nounlocking||| +sv_nv||5.005000| +sv_peek||5.005000| +sv_pos_b2u_midway||| +sv_pos_b2u||5.006000| +sv_pos_u2b_cached||| +sv_pos_u2b_forwards|||n +sv_pos_u2b_midway|||n +sv_pos_u2b||5.006000| +sv_pvbyten_force||5.006000| +sv_pvbyten||5.006000| +sv_pvbyte||5.006000| +sv_pvn_force_flags|5.007002||p +sv_pvn_force||| +sv_pvn_nomg|5.007003|5.005000|p +sv_pvn||5.005000| +sv_pvutf8n_force||5.006000| +sv_pvutf8n||5.006000| +sv_pvutf8||5.006000| +sv_pv||5.006000| +sv_recode_to_utf8||5.007003| +sv_reftype||| +sv_release_COW||| +sv_replace||| +sv_report_used||| +sv_reset||| +sv_rvweaken||5.006000| +sv_setiv_mg|5.004050||p +sv_setiv||| +sv_setnv_mg|5.006000||p +sv_setnv||| +sv_setpv_mg|5.004050||p +sv_setpvf_mg_nocontext|||pvn +sv_setpvf_mg|5.006000|5.004000|pv +sv_setpvf_nocontext|||vn +sv_setpvf||5.004000|v +sv_setpviv_mg||5.008001| +sv_setpviv||5.008001| +sv_setpvn_mg|5.004050||p +sv_setpvn||| +sv_setpvs|5.009004||p +sv_setpv||| +sv_setref_iv||| +sv_setref_nv||| +sv_setref_pvn||| +sv_setref_pv||| +sv_setref_uv||5.007001| +sv_setsv_cow||| +sv_setsv_flags||5.007002| +sv_setsv_mg|5.004050||p +sv_setsv_nomg|5.007002||p +sv_setsv||| +sv_setuv_mg|5.004050||p +sv_setuv|5.004000||p +sv_tainted||5.004000| +sv_taint||5.004000| +sv_true||5.005000| +sv_unglob||| +sv_uni_display||5.007003| +sv_unmagic||| +sv_unref_flags||5.007001| +sv_unref||| +sv_untaint||5.004000| +sv_upgrade||| +sv_usepvn_flags||5.009004| +sv_usepvn_mg|5.004050||p +sv_usepvn||| +sv_utf8_decode||5.006000| +sv_utf8_downgrade||5.006000| +sv_utf8_encode||5.006000| +sv_utf8_upgrade_flags_grow||5.011000| +sv_utf8_upgrade_flags||5.007002| +sv_utf8_upgrade_nomg||5.007002| +sv_utf8_upgrade||5.007001| +sv_uv|5.005000||p +sv_vcatpvf_mg|5.006000|5.004000|p +sv_vcatpvfn||5.004000| +sv_vcatpvf|5.006000|5.004000|p +sv_vsetpvf_mg|5.006000|5.004000|p +sv_vsetpvfn||5.004000| +sv_vsetpvf|5.006000|5.004000|p +sv_xmlpeek||| +svtype||| +swallow_bom||| +swap_match_buff||| +swash_fetch||5.007002| +swash_get||| +swash_init||5.006000| +sys_init3||5.010000|n +sys_init||5.010000|n +sys_intern_clear||| +sys_intern_dup||| +sys_intern_init||| +sys_term||5.010000|n +taint_env||| +taint_proper||| +tmps_grow||5.006000| +toLOWER||| +toUPPER||| +to_byte_substr||| +to_uni_fold||5.007003| +to_uni_lower_lc||5.006000| +to_uni_lower||5.007003| +to_uni_title_lc||5.006000| +to_uni_title||5.007003| +to_uni_upper_lc||5.006000| +to_uni_upper||5.007003| +to_utf8_case||5.007003| +to_utf8_fold||5.007003| +to_utf8_lower||5.007003| +to_utf8_substr||| +to_utf8_title||5.007003| +to_utf8_upper||5.007003| +token_free||| +token_getmad||| +tokenize_use||| +tokeq||| +tokereport||| +too_few_arguments||| +too_many_arguments||| +uiv_2buf|||n +unlnk||| +unpack_rec||| +unpack_str||5.007003| +unpackstring||5.008001| +unshare_hek_or_pvn||| +unshare_hek||| +unsharepvn||5.004000| +unwind_handler_stack||| +update_debugger_info||| +upg_version||5.009005| +usage||| +utf16_to_utf8_reversed||5.006001| +utf16_to_utf8||5.006001| +utf8_distance||5.006000| +utf8_hop||5.006000| +utf8_length||5.007001| +utf8_mg_pos_cache_update||| +utf8_to_bytes||5.006001| +utf8_to_uvchr||5.007001| +utf8_to_uvuni||5.007001| +utf8n_to_uvchr||| +utf8n_to_uvuni||5.007001| +utilize||| +uvchr_to_utf8_flags||5.007003| +uvchr_to_utf8||| +uvuni_to_utf8_flags||5.007003| +uvuni_to_utf8||5.007001| +validate_suid||| +varname||| +vcmp||5.009000| +vcroak||5.006000| +vdeb||5.007003| +vdie_common||| +vdie_croak_common||| +vdie||| +vform||5.006000| +visit||| +vivify_defelem||| +vivify_ref||| +vload_module|5.006000||p +vmess||5.006000| +vnewSVpvf|5.006000|5.004000|p +vnormal||5.009002| +vnumify||5.009000| +vstringify||5.009000| +vverify||5.009003| +vwarner||5.006000| +vwarn||5.006000| +wait4pid||| +warn_nocontext|||vn +warner_nocontext|||vn +warner|5.006000|5.004000|pv +warn|||v +watch||| +whichsig||| +write_no_mem||| +write_to_stderr||| +xmldump_all||| +xmldump_attr||| +xmldump_eval||| +xmldump_form||| +xmldump_indent|||v +xmldump_packsubs||| +xmldump_sub||| +xmldump_vindent||| +yyerror||| +yylex||| +yyparse||| +yywarn||| +); + +if (exists $opt{'list-unsupported'}) { + my $f; + for $f (sort { lc $a cmp lc $b } keys %API) { + next unless $API{$f}{todo}; + print "$f ", '.'x(40-length($f)), " ", format_version($API{$f}{todo}), "\n"; + } + exit 0; +} + +# Scan for possible replacement candidates + +my(%replace, %need, %hints, %warnings, %depends); +my $replace = 0; +my($hint, $define, $function); + +sub find_api +{ + my $code = shift; + $code =~ s{ + / (?: \*[^*]*\*+(?:[^$ccs][^*]*\*+)* / | /[^\r\n]*) + | "[^"\\]*(?:\\.[^"\\]*)*" + | '[^'\\]*(?:\\.[^'\\]*)*' }{}egsx; + grep { exists $API{$_} } $code =~ /(\w+)/mg; +} + +while () { + if ($hint) { + my $h = $hint->[0] eq 'Hint' ? \%hints : \%warnings; + if (m{^\s*\*\s(.*?)\s*$}) { + for (@{$hint->[1]}) { + $h->{$_} ||= ''; # suppress warning with older perls + $h->{$_} .= "$1\n"; + } + } + else { undef $hint } + } + + $hint = [$1, [split /,?\s+/, $2]] + if m{^\s*$rccs\s+(Hint|Warning):\s+(\w+(?:,?\s+\w+)*)\s*$}; + + if ($define) { + if ($define->[1] =~ /\\$/) { + $define->[1] .= $_; + } + else { + if (exists $API{$define->[0]} && $define->[1] !~ /^DPPP_\(/) { + my @n = find_api($define->[1]); + push @{$depends{$define->[0]}}, @n if @n + } + undef $define; + } + } + + $define = [$1, $2] if m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(.*)}; + + if ($function) { + if (/^}/) { + if (exists $API{$function->[0]}) { + my @n = find_api($function->[1]); + push @{$depends{$function->[0]}}, @n if @n + } + undef $function; + } + else { + $function->[1] .= $_; + } + } + + $function = [$1, ''] if m{^DPPP_\(my_(\w+)\)}; + + $replace = $1 if m{^\s*$rccs\s+Replace:\s+(\d+)\s+$rcce\s*$}; + $replace{$2} = $1 if $replace and m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(\w+)}; + $replace{$2} = $1 if m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(\w+).*$rccs\s+Replace\s+$rcce}; + $replace{$1} = $2 if m{^\s*$rccs\s+Replace (\w+) with (\w+)\s+$rcce\s*$}; + + if (m{^\s*$rccs\s+(\w+(\s*,\s*\w+)*)\s+depends\s+on\s+(\w+(\s*,\s*\w+)*)\s+$rcce\s*$}) { + my @deps = map { s/\s+//g; $_ } split /,/, $3; + my $d; + for $d (map { s/\s+//g; $_ } split /,/, $1) { + push @{$depends{$d}}, @deps; + } + } + + $need{$1} = 1 if m{^#if\s+defined\(NEED_(\w+)(?:_GLOBAL)?\)}; +} + +for (values %depends) { + my %s; + $_ = [sort grep !$s{$_}++, @$_]; +} + +if (exists $opt{'api-info'}) { + my $f; + my $count = 0; + my $match = $opt{'api-info'} =~ m!^/(.*)/$! ? $1 : "^\Q$opt{'api-info'}\E\$"; + for $f (sort { lc $a cmp lc $b } keys %API) { + next unless $f =~ /$match/; + print "\n=== $f ===\n\n"; + my $info = 0; + if ($API{$f}{base} || $API{$f}{todo}) { + my $base = format_version($API{$f}{base} || $API{$f}{todo}); + print "Supported at least starting from perl-$base.\n"; + $info++; + } + if ($API{$f}{provided}) { + my $todo = $API{$f}{todo} ? format_version($API{$f}{todo}) : "5.003"; + print "Support by $ppport provided back to perl-$todo.\n"; + print "Support needs to be explicitly requested by NEED_$f.\n" if exists $need{$f}; + print "Depends on: ", join(', ', @{$depends{$f}}), ".\n" if exists $depends{$f}; + print "\n$hints{$f}" if exists $hints{$f}; + print "\nWARNING:\n$warnings{$f}" if exists $warnings{$f}; + $info++; + } + print "No portability information available.\n" unless $info; + $count++; + } + $count or print "Found no API matching '$opt{'api-info'}'."; + print "\n"; + exit 0; +} + +if (exists $opt{'list-provided'}) { + my $f; + for $f (sort { lc $a cmp lc $b } keys %API) { + next unless $API{$f}{provided}; + my @flags; + push @flags, 'explicit' if exists $need{$f}; + push @flags, 'depend' if exists $depends{$f}; + push @flags, 'hint' if exists $hints{$f}; + push @flags, 'warning' if exists $warnings{$f}; + my $flags = @flags ? ' ['.join(', ', @flags).']' : ''; + print "$f$flags\n"; + } + exit 0; +} + +my @files; +my @srcext = qw( .xs .c .h .cc .cpp -c.inc -xs.inc ); +my $srcext = join '|', map { quotemeta $_ } @srcext; + +if (@ARGV) { + my %seen; + for (@ARGV) { + if (-e) { + if (-f) { + push @files, $_ unless $seen{$_}++; + } + else { warn "'$_' is not a file.\n" } + } + else { + my @new = grep { -f } glob $_ + or warn "'$_' does not exist.\n"; + push @files, grep { !$seen{$_}++ } @new; + } + } +} +else { + eval { + require File::Find; + File::Find::find(sub { + $File::Find::name =~ /($srcext)$/i + and push @files, $File::Find::name; + }, '.'); + }; + if ($@) { + @files = map { glob "*$_" } @srcext; + } +} + +if (!@ARGV || $opt{filter}) { + my(@in, @out); + my %xsc = map { /(.*)\.xs$/ ? ("$1.c" => 1, "$1.cc" => 1) : () } @files; + for (@files) { + my $out = exists $xsc{$_} || /\b\Q$ppport\E$/i || !/($srcext)$/i; + push @{ $out ? \@out : \@in }, $_; + } + if (@ARGV && @out) { + warning("Skipping the following files (use --nofilter to avoid this):\n| ", join "\n| ", @out); + } + @files = @in; +} + +die "No input files given!\n" unless @files; + +my(%files, %global, %revreplace); +%revreplace = reverse %replace; +my $filename; +my $patch_opened = 0; + +for $filename (@files) { + unless (open IN, "<$filename") { + warn "Unable to read from $filename: $!\n"; + next; + } + + info("Scanning $filename ..."); + + my $c = do { local $/; }; + close IN; + + my %file = (orig => $c, changes => 0); + + # Temporarily remove C/XS comments and strings from the code + my @ccom; + + $c =~ s{ + ( ^$HS*\#$HS*include\b[^\r\n]+\b(?:\Q$ppport\E|XSUB\.h)\b[^\r\n]* + | ^$HS*\#$HS*(?:define|elif|if(?:def)?)\b[^\r\n]* ) + | ( ^$HS*\#[^\r\n]* + | "[^"\\]*(?:\\.[^"\\]*)*" + | '[^'\\]*(?:\\.[^'\\]*)*' + | / (?: \*[^*]*\*+(?:[^$ccs][^*]*\*+)* / | /[^\r\n]* ) ) + }{ defined $2 and push @ccom, $2; + defined $1 ? $1 : "$ccs$#ccom$cce" }mgsex; + + $file{ccom} = \@ccom; + $file{code} = $c; + $file{has_inc_ppport} = $c =~ /^$HS*#$HS*include[^\r\n]+\b\Q$ppport\E\b/m; + + my $func; + + for $func (keys %API) { + my $match = $func; + $match .= "|$revreplace{$func}" if exists $revreplace{$func}; + if ($c =~ /\b(?:Perl_)?($match)\b/) { + $file{uses_replace}{$1}++ if exists $revreplace{$func} && $1 eq $revreplace{$func}; + $file{uses_Perl}{$func}++ if $c =~ /\bPerl_$func\b/; + if (exists $API{$func}{provided}) { + $file{uses_provided}{$func}++; + if (!exists $API{$func}{base} || $API{$func}{base} > $opt{'compat-version'}) { + $file{uses}{$func}++; + my @deps = rec_depend($func); + if (@deps) { + $file{uses_deps}{$func} = \@deps; + for (@deps) { + $file{uses}{$_} = 0 unless exists $file{uses}{$_}; + } + } + for ($func, @deps) { + $file{needs}{$_} = 'static' if exists $need{$_}; + } + } + } + if (exists $API{$func}{todo} && $API{$func}{todo} > $opt{'compat-version'}) { + if ($c =~ /\b$func\b/) { + $file{uses_todo}{$func}++; + } + } + } + } + + while ($c =~ /^$HS*#$HS*define$HS+(NEED_(\w+?)(_GLOBAL)?)\b/mg) { + if (exists $need{$2}) { + $file{defined $3 ? 'needed_global' : 'needed_static'}{$2}++; + } + else { warning("Possibly wrong #define $1 in $filename") } + } + + for (qw(uses needs uses_todo needed_global needed_static)) { + for $func (keys %{$file{$_}}) { + push @{$global{$_}{$func}}, $filename; + } + } + + $files{$filename} = \%file; +} + +# Globally resolve NEED_'s +my $need; +for $need (keys %{$global{needs}}) { + if (@{$global{needs}{$need}} > 1) { + my @targets = @{$global{needs}{$need}}; + my @t = grep $files{$_}{needed_global}{$need}, @targets; + @targets = @t if @t; + @t = grep /\.xs$/i, @targets; + @targets = @t if @t; + my $target = shift @targets; + $files{$target}{needs}{$need} = 'global'; + for (@{$global{needs}{$need}}) { + $files{$_}{needs}{$need} = 'extern' if $_ ne $target; + } + } +} + +for $filename (@files) { + exists $files{$filename} or next; + + info("=== Analyzing $filename ==="); + + my %file = %{$files{$filename}}; + my $func; + my $c = $file{code}; + my $warnings = 0; + + for $func (sort keys %{$file{uses_Perl}}) { + if ($API{$func}{varargs}) { + unless ($API{$func}{nothxarg}) { + my $changes = ($c =~ s{\b(Perl_$func\s*\(\s*)(?!aTHX_?)(\)|[^\s)]*\))} + { $1 . ($2 eq ')' ? 'aTHX' : 'aTHX_ ') . $2 }ge); + if ($changes) { + warning("Doesn't pass interpreter argument aTHX to Perl_$func"); + $file{changes} += $changes; + } + } + } + else { + warning("Uses Perl_$func instead of $func"); + $file{changes} += ($c =~ s{\bPerl_$func(\s*)\((\s*aTHX_?)?\s*} + {$func$1(}g); + } + } + + for $func (sort keys %{$file{uses_replace}}) { + warning("Uses $func instead of $replace{$func}"); + $file{changes} += ($c =~ s/\b$func\b/$replace{$func}/g); + } + + for $func (sort keys %{$file{uses_provided}}) { + if ($file{uses}{$func}) { + if (exists $file{uses_deps}{$func}) { + diag("Uses $func, which depends on ", join(', ', @{$file{uses_deps}{$func}})); + } + else { + diag("Uses $func"); + } + } + $warnings += hint($func); + } + + unless ($opt{quiet}) { + for $func (sort keys %{$file{uses_todo}}) { + print "*** WARNING: Uses $func, which may not be portable below perl ", + format_version($API{$func}{todo}), ", even with '$ppport'\n"; + $warnings++; + } + } + + for $func (sort keys %{$file{needed_static}}) { + my $message = ''; + if (not exists $file{uses}{$func}) { + $message = "No need to define NEED_$func if $func is never used"; + } + elsif (exists $file{needs}{$func} && $file{needs}{$func} ne 'static') { + $message = "No need to define NEED_$func when already needed globally"; + } + if ($message) { + diag($message); + $file{changes} += ($c =~ s/^$HS*#$HS*define$HS+NEED_$func\b.*$LF//mg); + } + } + + for $func (sort keys %{$file{needed_global}}) { + my $message = ''; + if (not exists $global{uses}{$func}) { + $message = "No need to define NEED_${func}_GLOBAL if $func is never used"; + } + elsif (exists $file{needs}{$func}) { + if ($file{needs}{$func} eq 'extern') { + $message = "No need to define NEED_${func}_GLOBAL when already needed globally"; + } + elsif ($file{needs}{$func} eq 'static') { + $message = "No need to define NEED_${func}_GLOBAL when only used in this file"; + } + } + if ($message) { + diag($message); + $file{changes} += ($c =~ s/^$HS*#$HS*define$HS+NEED_${func}_GLOBAL\b.*$LF//mg); + } + } + + $file{needs_inc_ppport} = keys %{$file{uses}}; + + if ($file{needs_inc_ppport}) { + my $pp = ''; + + for $func (sort keys %{$file{needs}}) { + my $type = $file{needs}{$func}; + next if $type eq 'extern'; + my $suffix = $type eq 'global' ? '_GLOBAL' : ''; + unless (exists $file{"needed_$type"}{$func}) { + if ($type eq 'global') { + diag("Files [@{$global{needs}{$func}}] need $func, adding global request"); + } + else { + diag("File needs $func, adding static request"); + } + $pp .= "#define NEED_$func$suffix\n"; + } + } + + if ($pp && ($c =~ s/^(?=$HS*#$HS*define$HS+NEED_\w+)/$pp/m)) { + $pp = ''; + $file{changes}++; + } + + unless ($file{has_inc_ppport}) { + diag("Needs to include '$ppport'"); + $pp .= qq(#include "$ppport"\n) + } + + if ($pp) { + $file{changes} += ($c =~ s/^($HS*#$HS*define$HS+NEED_\w+.*?)^/$1$pp/ms) + || ($c =~ s/^(?=$HS*#$HS*include.*\Q$ppport\E)/$pp/m) + || ($c =~ s/^($HS*#$HS*include.*XSUB.*\s*?)^/$1$pp/m) + || ($c =~ s/^/$pp/); + } + } + else { + if ($file{has_inc_ppport}) { + diag("No need to include '$ppport'"); + $file{changes} += ($c =~ s/^$HS*?#$HS*include.*\Q$ppport\E.*?$LF//m); + } + } + + # put back in our C comments + my $ix; + my $cppc = 0; + my @ccom = @{$file{ccom}}; + for $ix (0 .. $#ccom) { + if (!$opt{cplusplus} && $ccom[$ix] =~ s!^//!!) { + $cppc++; + $file{changes} += $c =~ s/$rccs$ix$rcce/$ccs$ccom[$ix] $cce/; + } + else { + $c =~ s/$rccs$ix$rcce/$ccom[$ix]/; + } + } + + if ($cppc) { + my $s = $cppc != 1 ? 's' : ''; + warning("Uses $cppc C++ style comment$s, which is not portable"); + } + + my $s = $warnings != 1 ? 's' : ''; + my $warn = $warnings ? " ($warnings warning$s)" : ''; + info("Analysis completed$warn"); + + if ($file{changes}) { + if (exists $opt{copy}) { + my $newfile = "$filename$opt{copy}"; + if (-e $newfile) { + error("'$newfile' already exists, refusing to write copy of '$filename'"); + } + else { + local *F; + if (open F, ">$newfile") { + info("Writing copy of '$filename' with changes to '$newfile'"); + print F $c; + close F; + } + else { + error("Cannot open '$newfile' for writing: $!"); + } + } + } + elsif (exists $opt{patch} || $opt{changes}) { + if (exists $opt{patch}) { + unless ($patch_opened) { + if (open PATCH, ">$opt{patch}") { + $patch_opened = 1; + } + else { + error("Cannot open '$opt{patch}' for writing: $!"); + delete $opt{patch}; + $opt{changes} = 1; + goto fallback; + } + } + mydiff(\*PATCH, $filename, $c); + } + else { +fallback: + info("Suggested changes:"); + mydiff(\*STDOUT, $filename, $c); + } + } + else { + my $s = $file{changes} == 1 ? '' : 's'; + info("$file{changes} potentially required change$s detected"); + } + } + else { + info("Looks good"); + } +} + +close PATCH if $patch_opened; + +exit 0; + + +sub try_use { eval "use @_;"; return $@ eq '' } + +sub mydiff +{ + local *F = shift; + my($file, $str) = @_; + my $diff; + + if (exists $opt{diff}) { + $diff = run_diff($opt{diff}, $file, $str); + } + + if (!defined $diff and try_use('Text::Diff')) { + $diff = Text::Diff::diff($file, \$str, { STYLE => 'Unified' }); + $diff = <
$tmp") { + print F $str; + close F; + + if (open F, "$prog $file $tmp |") { + while () { + s/\Q$tmp\E/$file.patched/; + $diff .= $_; + } + close F; + unlink $tmp; + return $diff; + } + + unlink $tmp; + } + else { + error("Cannot open '$tmp' for writing: $!"); + } + + return undef; +} + +sub rec_depend +{ + my($func, $seen) = @_; + return () unless exists $depends{$func}; + $seen = {%{$seen||{}}}; + return () if $seen->{$func}++; + my %s; + grep !$s{$_}++, map { ($_, rec_depend($_, $seen)) } @{$depends{$func}}; +} + +sub parse_version +{ + my $ver = shift; + + if ($ver =~ /^(\d+)\.(\d+)\.(\d+)$/) { + return ($1, $2, $3); + } + elsif ($ver !~ /^\d+\.[\d_]+$/) { + die "cannot parse version '$ver'\n"; + } + + $ver =~ s/_//g; + $ver =~ s/$/000000/; + + my($r,$v,$s) = $ver =~ /(\d+)\.(\d{3})(\d{3})/; + + $v = int $v; + $s = int $s; + + if ($r < 5 || ($r == 5 && $v < 6)) { + if ($s % 10) { + die "cannot parse version '$ver'\n"; + } + } + + return ($r, $v, $s); +} + +sub format_version +{ + my $ver = shift; + + $ver =~ s/$/000000/; + my($r,$v,$s) = $ver =~ /(\d+)\.(\d{3})(\d{3})/; + + $v = int $v; + $s = int $s; + + if ($r < 5 || ($r == 5 && $v < 6)) { + if ($s % 10) { + die "invalid version '$ver'\n"; + } + $s /= 10; + + $ver = sprintf "%d.%03d", $r, $v; + $s > 0 and $ver .= sprintf "_%02d", $s; + + return $ver; + } + + return sprintf "%d.%d.%d", $r, $v, $s; +} + +sub info +{ + $opt{quiet} and return; + print @_, "\n"; +} + +sub diag +{ + $opt{quiet} and return; + $opt{diag} and print @_, "\n"; +} + +sub warning +{ + $opt{quiet} and return; + print "*** ", @_, "\n"; +} + +sub error +{ + print "*** ERROR: ", @_, "\n"; +} + +my %given_hints; +my %given_warnings; +sub hint +{ + $opt{quiet} and return; + my $func = shift; + my $rv = 0; + if (exists $warnings{$func} && !$given_warnings{$func}++) { + my $warn = $warnings{$func}; + $warn =~ s!^!*** !mg; + print "*** WARNING: $func\n", $warn; + $rv++; + } + if ($opt{hints} && exists $hints{$func} && !$given_hints{$func}++) { + my $hint = $hints{$func}; + $hint =~ s/^/ /mg; + print " --- hint for $func ---\n", $hint; + } + $rv; +} + +sub usage +{ + my($usage) = do { local(@ARGV,$/)=($0); <> } =~ /^=head\d$HS+SYNOPSIS\s*^(.*?)\s*^=/ms; + my %M = ( 'I' => '*' ); + $usage =~ s/^\s*perl\s+\S+/$^X $0/; + $usage =~ s/([A-Z])<([^>]+)>/$M{$1}$2$M{$1}/g; + + print < }; + my($copy) = $self =~ /^=head\d\s+COPYRIGHT\s*^(.*?)^=\w+/ms; + $copy =~ s/^(?=\S+)/ /gms; + $self =~ s/^$HS+Do NOT edit.*?(?=^-)/$copy/ms; + $self =~ s/^SKIP.*(?=^__DATA__)/SKIP +if (\@ARGV && \$ARGV[0] eq '--unstrip') { + eval { require Devel::PPPort }; + \$@ and die "Cannot require Devel::PPPort, please install.\\n"; + if (eval \$Devel::PPPort::VERSION < $VERSION) { + die "$0 was originally generated with Devel::PPPort $VERSION.\\n" + . "Your Devel::PPPort is only version \$Devel::PPPort::VERSION.\\n" + . "Please install a newer version, or --unstrip will not work.\\n"; + } + Devel::PPPort::WriteFile(\$0); + exit 0; +} +print <$0" or die "cannot strip $0: $!\n"; + print OUT "$pl$c\n"; + + exit 0; +} + +__DATA__ +*/ + +#ifndef _P_P_PORTABILITY_H_ +#define _P_P_PORTABILITY_H_ + +#ifndef DPPP_NAMESPACE +# define DPPP_NAMESPACE DPPP_ +#endif + +#define DPPP_CAT2(x,y) CAT2(x,y) +#define DPPP_(name) DPPP_CAT2(DPPP_NAMESPACE, name) + +#ifndef PERL_REVISION +# if !defined(__PATCHLEVEL_H_INCLUDED__) && !(defined(PATCHLEVEL) && defined(SUBVERSION)) +# define PERL_PATCHLEVEL_H_IMPLICIT +# include +# endif +# if !(defined(PERL_VERSION) || (defined(SUBVERSION) && defined(PATCHLEVEL))) +# include +# endif +# ifndef PERL_REVISION +# define PERL_REVISION (5) + /* Replace: 1 */ +# define PERL_VERSION PATCHLEVEL +# define PERL_SUBVERSION SUBVERSION + /* Replace PERL_PATCHLEVEL with PERL_VERSION */ + /* Replace: 0 */ +# endif +#endif + +#define _dpppDEC2BCD(dec) ((((dec)/100)<<8)|((((dec)%100)/10)<<4)|((dec)%10)) +#define PERL_BCDVERSION ((_dpppDEC2BCD(PERL_REVISION)<<24)|(_dpppDEC2BCD(PERL_VERSION)<<12)|_dpppDEC2BCD(PERL_SUBVERSION)) + +/* It is very unlikely that anyone will try to use this with Perl 6 + (or greater), but who knows. + */ +#if PERL_REVISION != 5 +# error ppport.h only works with Perl version 5 +#endif /* PERL_REVISION != 5 */ +#ifndef dTHR +# define dTHR dNOOP +#endif +#ifndef dTHX +# define dTHX dNOOP +#endif + +#ifndef dTHXa +# define dTHXa(x) dNOOP +#endif +#ifndef pTHX +# define pTHX void +#endif + +#ifndef pTHX_ +# define pTHX_ +#endif + +#ifndef aTHX +# define aTHX +#endif + +#ifndef aTHX_ +# define aTHX_ +#endif + +#if (PERL_BCDVERSION < 0x5006000) +# ifdef USE_THREADS +# define aTHXR thr +# define aTHXR_ thr, +# else +# define aTHXR +# define aTHXR_ +# endif +# define dTHXR dTHR +#else +# define aTHXR aTHX +# define aTHXR_ aTHX_ +# define dTHXR dTHX +#endif +#ifndef dTHXoa +# define dTHXoa(x) dTHXa(x) +#endif + +#ifdef I_LIMITS +# include +#endif + +#ifndef PERL_UCHAR_MIN +# define PERL_UCHAR_MIN ((unsigned char)0) +#endif + +#ifndef PERL_UCHAR_MAX +# ifdef UCHAR_MAX +# define PERL_UCHAR_MAX ((unsigned char)UCHAR_MAX) +# else +# ifdef MAXUCHAR +# define PERL_UCHAR_MAX ((unsigned char)MAXUCHAR) +# else +# define PERL_UCHAR_MAX ((unsigned char)~(unsigned)0) +# endif +# endif +#endif + +#ifndef PERL_USHORT_MIN +# define PERL_USHORT_MIN ((unsigned short)0) +#endif + +#ifndef PERL_USHORT_MAX +# ifdef USHORT_MAX +# define PERL_USHORT_MAX ((unsigned short)USHORT_MAX) +# else +# ifdef MAXUSHORT +# define PERL_USHORT_MAX ((unsigned short)MAXUSHORT) +# else +# ifdef USHRT_MAX +# define PERL_USHORT_MAX ((unsigned short)USHRT_MAX) +# else +# define PERL_USHORT_MAX ((unsigned short)~(unsigned)0) +# endif +# endif +# endif +#endif + +#ifndef PERL_SHORT_MAX +# ifdef SHORT_MAX +# define PERL_SHORT_MAX ((short)SHORT_MAX) +# else +# ifdef MAXSHORT /* Often used in */ +# define PERL_SHORT_MAX ((short)MAXSHORT) +# else +# ifdef SHRT_MAX +# define PERL_SHORT_MAX ((short)SHRT_MAX) +# else +# define PERL_SHORT_MAX ((short) (PERL_USHORT_MAX >> 1)) +# endif +# endif +# endif +#endif + +#ifndef PERL_SHORT_MIN +# ifdef SHORT_MIN +# define PERL_SHORT_MIN ((short)SHORT_MIN) +# else +# ifdef MINSHORT +# define PERL_SHORT_MIN ((short)MINSHORT) +# else +# ifdef SHRT_MIN +# define PERL_SHORT_MIN ((short)SHRT_MIN) +# else +# define PERL_SHORT_MIN (-PERL_SHORT_MAX - ((3 & -1) == 3)) +# endif +# endif +# endif +#endif + +#ifndef PERL_UINT_MAX +# ifdef UINT_MAX +# define PERL_UINT_MAX ((unsigned int)UINT_MAX) +# else +# ifdef MAXUINT +# define PERL_UINT_MAX ((unsigned int)MAXUINT) +# else +# define PERL_UINT_MAX (~(unsigned int)0) +# endif +# endif +#endif + +#ifndef PERL_UINT_MIN +# define PERL_UINT_MIN ((unsigned int)0) +#endif + +#ifndef PERL_INT_MAX +# ifdef INT_MAX +# define PERL_INT_MAX ((int)INT_MAX) +# else +# ifdef MAXINT /* Often used in */ +# define PERL_INT_MAX ((int)MAXINT) +# else +# define PERL_INT_MAX ((int)(PERL_UINT_MAX >> 1)) +# endif +# endif +#endif + +#ifndef PERL_INT_MIN +# ifdef INT_MIN +# define PERL_INT_MIN ((int)INT_MIN) +# else +# ifdef MININT +# define PERL_INT_MIN ((int)MININT) +# else +# define PERL_INT_MIN (-PERL_INT_MAX - ((3 & -1) == 3)) +# endif +# endif +#endif + +#ifndef PERL_ULONG_MAX +# ifdef ULONG_MAX +# define PERL_ULONG_MAX ((unsigned long)ULONG_MAX) +# else +# ifdef MAXULONG +# define PERL_ULONG_MAX ((unsigned long)MAXULONG) +# else +# define PERL_ULONG_MAX (~(unsigned long)0) +# endif +# endif +#endif + +#ifndef PERL_ULONG_MIN +# define PERL_ULONG_MIN ((unsigned long)0L) +#endif + +#ifndef PERL_LONG_MAX +# ifdef LONG_MAX +# define PERL_LONG_MAX ((long)LONG_MAX) +# else +# ifdef MAXLONG +# define PERL_LONG_MAX ((long)MAXLONG) +# else +# define PERL_LONG_MAX ((long) (PERL_ULONG_MAX >> 1)) +# endif +# endif +#endif + +#ifndef PERL_LONG_MIN +# ifdef LONG_MIN +# define PERL_LONG_MIN ((long)LONG_MIN) +# else +# ifdef MINLONG +# define PERL_LONG_MIN ((long)MINLONG) +# else +# define PERL_LONG_MIN (-PERL_LONG_MAX - ((3 & -1) == 3)) +# endif +# endif +#endif + +#if defined(HAS_QUAD) && (defined(convex) || defined(uts)) +# ifndef PERL_UQUAD_MAX +# ifdef ULONGLONG_MAX +# define PERL_UQUAD_MAX ((unsigned long long)ULONGLONG_MAX) +# else +# ifdef MAXULONGLONG +# define PERL_UQUAD_MAX ((unsigned long long)MAXULONGLONG) +# else +# define PERL_UQUAD_MAX (~(unsigned long long)0) +# endif +# endif +# endif + +# ifndef PERL_UQUAD_MIN +# define PERL_UQUAD_MIN ((unsigned long long)0L) +# endif + +# ifndef PERL_QUAD_MAX +# ifdef LONGLONG_MAX +# define PERL_QUAD_MAX ((long long)LONGLONG_MAX) +# else +# ifdef MAXLONGLONG +# define PERL_QUAD_MAX ((long long)MAXLONGLONG) +# else +# define PERL_QUAD_MAX ((long long) (PERL_UQUAD_MAX >> 1)) +# endif +# endif +# endif + +# ifndef PERL_QUAD_MIN +# ifdef LONGLONG_MIN +# define PERL_QUAD_MIN ((long long)LONGLONG_MIN) +# else +# ifdef MINLONGLONG +# define PERL_QUAD_MIN ((long long)MINLONGLONG) +# else +# define PERL_QUAD_MIN (-PERL_QUAD_MAX - ((3 & -1) == 3)) +# endif +# endif +# endif +#endif + +/* This is based on code from 5.003 perl.h */ +#ifdef HAS_QUAD +# ifdef cray +#ifndef IVTYPE +# define IVTYPE int +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_INT_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_INT_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_UINT_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_UINT_MAX +#endif + +# ifdef INTSIZE +#ifndef IVSIZE +# define IVSIZE INTSIZE +#endif + +# endif +# else +# if defined(convex) || defined(uts) +#ifndef IVTYPE +# define IVTYPE long long +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_QUAD_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_QUAD_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_UQUAD_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_UQUAD_MAX +#endif + +# ifdef LONGLONGSIZE +#ifndef IVSIZE +# define IVSIZE LONGLONGSIZE +#endif + +# endif +# else +#ifndef IVTYPE +# define IVTYPE long +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_LONG_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_LONG_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_ULONG_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_ULONG_MAX +#endif + +# ifdef LONGSIZE +#ifndef IVSIZE +# define IVSIZE LONGSIZE +#endif + +# endif +# endif +# endif +#ifndef IVSIZE +# define IVSIZE 8 +#endif + +#ifndef PERL_QUAD_MIN +# define PERL_QUAD_MIN IV_MIN +#endif + +#ifndef PERL_QUAD_MAX +# define PERL_QUAD_MAX IV_MAX +#endif + +#ifndef PERL_UQUAD_MIN +# define PERL_UQUAD_MIN UV_MIN +#endif + +#ifndef PERL_UQUAD_MAX +# define PERL_UQUAD_MAX UV_MAX +#endif + +#else +#ifndef IVTYPE +# define IVTYPE long +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_LONG_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_LONG_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_ULONG_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_ULONG_MAX +#endif + +#endif + +#ifndef IVSIZE +# ifdef LONGSIZE +# define IVSIZE LONGSIZE +# else +# define IVSIZE 4 /* A bold guess, but the best we can make. */ +# endif +#endif +#ifndef UVTYPE +# define UVTYPE unsigned IVTYPE +#endif + +#ifndef UVSIZE +# define UVSIZE IVSIZE +#endif +#ifndef sv_setuv +# define sv_setuv(sv, uv) \ + STMT_START { \ + UV TeMpUv = uv; \ + if (TeMpUv <= IV_MAX) \ + sv_setiv(sv, TeMpUv); \ + else \ + sv_setnv(sv, (double)TeMpUv); \ + } STMT_END +#endif +#ifndef newSVuv +# define newSVuv(uv) ((uv) <= IV_MAX ? newSViv((IV)uv) : newSVnv((NV)uv)) +#endif +#ifndef sv_2uv +# define sv_2uv(sv) ((PL_Sv = (sv)), (UV) (SvNOK(PL_Sv) ? SvNV(PL_Sv) : sv_2nv(PL_Sv))) +#endif + +#ifndef SvUVX +# define SvUVX(sv) ((UV)SvIVX(sv)) +#endif + +#ifndef SvUVXx +# define SvUVXx(sv) SvUVX(sv) +#endif + +#ifndef SvUV +# define SvUV(sv) (SvIOK(sv) ? SvUVX(sv) : sv_2uv(sv)) +#endif + +#ifndef SvUVx +# define SvUVx(sv) ((PL_Sv = (sv)), SvUV(PL_Sv)) +#endif + +/* Hint: sv_uv + * Always use the SvUVx() macro instead of sv_uv(). + */ +#ifndef sv_uv +# define sv_uv(sv) SvUVx(sv) +#endif + +#if !defined(SvUOK) && defined(SvIOK_UV) +# define SvUOK(sv) SvIOK_UV(sv) +#endif +#ifndef XST_mUV +# define XST_mUV(i,v) (ST(i) = sv_2mortal(newSVuv(v)) ) +#endif + +#ifndef XSRETURN_UV +# define XSRETURN_UV(v) STMT_START { XST_mUV(0,v); XSRETURN(1); } STMT_END +#endif +#ifndef PUSHu +# define PUSHu(u) STMT_START { sv_setuv(TARG, (UV)(u)); PUSHTARG; } STMT_END +#endif + +#ifndef XPUSHu +# define XPUSHu(u) STMT_START { sv_setuv(TARG, (UV)(u)); XPUSHTARG; } STMT_END +#endif + +#ifdef HAS_MEMCMP +#ifndef memNE +# define memNE(s1,s2,l) (memcmp(s1,s2,l)) +#endif + +#ifndef memEQ +# define memEQ(s1,s2,l) (!memcmp(s1,s2,l)) +#endif + +#else +#ifndef memNE +# define memNE(s1,s2,l) (bcmp(s1,s2,l)) +#endif + +#ifndef memEQ +# define memEQ(s1,s2,l) (!bcmp(s1,s2,l)) +#endif + +#endif +#ifndef MoveD +# define MoveD(s,d,n,t) memmove((char*)(d),(char*)(s), (n) * sizeof(t)) +#endif + +#ifndef CopyD +# define CopyD(s,d,n,t) memcpy((char*)(d),(char*)(s), (n) * sizeof(t)) +#endif + +#ifdef HAS_MEMSET +#ifndef ZeroD +# define ZeroD(d,n,t) memzero((char*)(d), (n) * sizeof(t)) +#endif + +#else +#ifndef ZeroD +# define ZeroD(d,n,t) ((void)memzero((char*)(d), (n) * sizeof(t)), d) +#endif + +#endif +#ifndef PoisonWith +# define PoisonWith(d,n,t,b) (void)memset((char*)(d), (U8)(b), (n) * sizeof(t)) +#endif + +#ifndef PoisonNew +# define PoisonNew(d,n,t) PoisonWith(d,n,t,0xAB) +#endif + +#ifndef PoisonFree +# define PoisonFree(d,n,t) PoisonWith(d,n,t,0xEF) +#endif + +#ifndef Poison +# define Poison(d,n,t) PoisonFree(d,n,t) +#endif +#ifndef Newx +# define Newx(v,n,t) New(0,v,n,t) +#endif + +#ifndef Newxc +# define Newxc(v,n,t,c) Newc(0,v,n,t,c) +#endif + +#ifndef Newxz +# define Newxz(v,n,t) Newz(0,v,n,t) +#endif + +#ifndef PERL_UNUSED_DECL +# ifdef HASATTRIBUTE +# if (defined(__GNUC__) && defined(__cplusplus)) || defined(__INTEL_COMPILER) +# define PERL_UNUSED_DECL +# else +# define PERL_UNUSED_DECL __attribute__((unused)) +# endif +# else +# define PERL_UNUSED_DECL +# endif +#endif + +#ifndef PERL_UNUSED_ARG +# if defined(lint) && defined(S_SPLINT_S) /* www.splint.org */ +# include +# define PERL_UNUSED_ARG(x) NOTE(ARGUNUSED(x)) +# else +# define PERL_UNUSED_ARG(x) ((void)x) +# endif +#endif + +#ifndef PERL_UNUSED_VAR +# define PERL_UNUSED_VAR(x) ((void)x) +#endif + +#ifndef PERL_UNUSED_CONTEXT +# ifdef USE_ITHREADS +# define PERL_UNUSED_CONTEXT PERL_UNUSED_ARG(my_perl) +# else +# define PERL_UNUSED_CONTEXT +# endif +#endif +#ifndef NOOP +# define NOOP /*EMPTY*/(void)0 +#endif + +#ifndef dNOOP +# define dNOOP extern int /*@unused@*/ Perl___notused PERL_UNUSED_DECL +#endif + +#ifndef NVTYPE +# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) +# define NVTYPE long double +# else +# define NVTYPE double +# endif +typedef NVTYPE NV; +#endif + +#ifndef INT2PTR +# if (IVSIZE == PTRSIZE) && (UVSIZE == PTRSIZE) +# define PTRV UV +# define INT2PTR(any,d) (any)(d) +# else +# if PTRSIZE == LONGSIZE +# define PTRV unsigned long +# else +# define PTRV unsigned +# endif +# define INT2PTR(any,d) (any)(PTRV)(d) +# endif +#endif + +#ifndef PTR2ul +# if PTRSIZE == LONGSIZE +# define PTR2ul(p) (unsigned long)(p) +# else +# define PTR2ul(p) INT2PTR(unsigned long,p) +# endif +#endif +#ifndef PTR2nat +# define PTR2nat(p) (PTRV)(p) +#endif + +#ifndef NUM2PTR +# define NUM2PTR(any,d) (any)PTR2nat(d) +#endif + +#ifndef PTR2IV +# define PTR2IV(p) INT2PTR(IV,p) +#endif + +#ifndef PTR2UV +# define PTR2UV(p) INT2PTR(UV,p) +#endif + +#ifndef PTR2NV +# define PTR2NV(p) NUM2PTR(NV,p) +#endif + +#undef START_EXTERN_C +#undef END_EXTERN_C +#undef EXTERN_C +#ifdef __cplusplus +# define START_EXTERN_C extern "C" { +# define END_EXTERN_C } +# define EXTERN_C extern "C" +#else +# define START_EXTERN_C +# define END_EXTERN_C +# define EXTERN_C extern +#endif + +#if defined(PERL_GCC_PEDANTIC) +# ifndef PERL_GCC_BRACE_GROUPS_FORBIDDEN +# define PERL_GCC_BRACE_GROUPS_FORBIDDEN +# endif +#endif + +#if defined(__GNUC__) && !defined(PERL_GCC_BRACE_GROUPS_FORBIDDEN) && !defined(__cplusplus) +# ifndef PERL_USE_GCC_BRACE_GROUPS +# define PERL_USE_GCC_BRACE_GROUPS +# endif +#endif + +#undef STMT_START +#undef STMT_END +#ifdef PERL_USE_GCC_BRACE_GROUPS +# define STMT_START (void)( /* gcc supports ``({ STATEMENTS; })'' */ +# define STMT_END ) +#else +# if defined(VOIDFLAGS) && (VOIDFLAGS) && (defined(sun) || defined(__sun__)) && !defined(__GNUC__) +# define STMT_START if (1) +# define STMT_END else (void)0 +# else +# define STMT_START do +# define STMT_END while (0) +# endif +#endif +#ifndef boolSV +# define boolSV(b) ((b) ? &PL_sv_yes : &PL_sv_no) +#endif + +/* DEFSV appears first in 5.004_56 */ +#ifndef DEFSV +# define DEFSV GvSV(PL_defgv) +#endif + +#ifndef SAVE_DEFSV +# define SAVE_DEFSV SAVESPTR(GvSV(PL_defgv)) +#endif + +#ifndef DEFSV_set +# define DEFSV_set(sv) (DEFSV = (sv)) +#endif + +/* Older perls (<=5.003) lack AvFILLp */ +#ifndef AvFILLp +# define AvFILLp AvFILL +#endif +#ifndef ERRSV +# define ERRSV get_sv("@",FALSE) +#endif + +/* Hint: gv_stashpvn + * This function's backport doesn't support the length parameter, but + * rather ignores it. Portability can only be ensured if the length + * parameter is used for speed reasons, but the length can always be + * correctly computed from the string argument. + */ +#ifndef gv_stashpvn +# define gv_stashpvn(str,len,create) gv_stashpv(str,create) +#endif + +/* Replace: 1 */ +#ifndef get_cv +# define get_cv perl_get_cv +#endif + +#ifndef get_sv +# define get_sv perl_get_sv +#endif + +#ifndef get_av +# define get_av perl_get_av +#endif + +#ifndef get_hv +# define get_hv perl_get_hv +#endif + +/* Replace: 0 */ +#ifndef dUNDERBAR +# define dUNDERBAR dNOOP +#endif + +#ifndef UNDERBAR +# define UNDERBAR DEFSV +#endif +#ifndef dAX +# define dAX I32 ax = MARK - PL_stack_base + 1 +#endif + +#ifndef dITEMS +# define dITEMS I32 items = SP - MARK +#endif +#ifndef dXSTARG +# define dXSTARG SV * targ = sv_newmortal() +#endif +#ifndef dAXMARK +# define dAXMARK I32 ax = POPMARK; \ + register SV ** const mark = PL_stack_base + ax++ +#endif +#ifndef XSprePUSH +# define XSprePUSH (sp = PL_stack_base + ax - 1) +#endif + +#if (PERL_BCDVERSION < 0x5005000) +# undef XSRETURN +# define XSRETURN(off) \ + STMT_START { \ + PL_stack_sp = PL_stack_base + ax + ((off) - 1); \ + return; \ + } STMT_END +#endif +#ifndef PERL_ABS +# define PERL_ABS(x) ((x) < 0 ? -(x) : (x)) +#endif +#ifndef dVAR +# define dVAR dNOOP +#endif +#ifndef SVf +# define SVf "_" +#endif +#ifndef UTF8_MAXBYTES +# define UTF8_MAXBYTES UTF8_MAXLEN +#endif +#ifndef CPERLscope +# define CPERLscope(x) x +#endif +#ifndef PERL_HASH +# define PERL_HASH(hash,str,len) \ + STMT_START { \ + const char *s_PeRlHaSh = str; \ + I32 i_PeRlHaSh = len; \ + U32 hash_PeRlHaSh = 0; \ + while (i_PeRlHaSh--) \ + hash_PeRlHaSh = hash_PeRlHaSh * 33 + *s_PeRlHaSh++; \ + (hash) = hash_PeRlHaSh; \ + } STMT_END +#endif + +#ifndef PERLIO_FUNCS_DECL +# ifdef PERLIO_FUNCS_CONST +# define PERLIO_FUNCS_DECL(funcs) const PerlIO_funcs funcs +# define PERLIO_FUNCS_CAST(funcs) (PerlIO_funcs*)(funcs) +# else +# define PERLIO_FUNCS_DECL(funcs) PerlIO_funcs funcs +# define PERLIO_FUNCS_CAST(funcs) (funcs) +# endif +#endif + +/* provide these typedefs for older perls */ +#if (PERL_BCDVERSION < 0x5009003) + +# ifdef ARGSproto +typedef OP* (CPERLscope(*Perl_ppaddr_t))(ARGSproto); +# else +typedef OP* (CPERLscope(*Perl_ppaddr_t))(pTHX); +# endif + +typedef OP* (CPERLscope(*Perl_check_t)) (pTHX_ OP*); + +#endif +#ifndef isPSXSPC +# define isPSXSPC(c) (isSPACE(c) || (c) == '\v') +#endif + +#ifndef isBLANK +# define isBLANK(c) ((c) == ' ' || (c) == '\t') +#endif + +#ifdef EBCDIC +#ifndef isALNUMC +# define isALNUMC(c) isalnum(c) +#endif + +#ifndef isASCII +# define isASCII(c) isascii(c) +#endif + +#ifndef isCNTRL +# define isCNTRL(c) iscntrl(c) +#endif + +#ifndef isGRAPH +# define isGRAPH(c) isgraph(c) +#endif + +#ifndef isPRINT +# define isPRINT(c) isprint(c) +#endif + +#ifndef isPUNCT +# define isPUNCT(c) ispunct(c) +#endif + +#ifndef isXDIGIT +# define isXDIGIT(c) isxdigit(c) +#endif + +#else +# if (PERL_BCDVERSION < 0x5010000) +/* Hint: isPRINT + * The implementation in older perl versions includes all of the + * isSPACE() characters, which is wrong. The version provided by + * Devel::PPPort always overrides a present buggy version. + */ +# undef isPRINT +# endif +#ifndef isALNUMC +# define isALNUMC(c) (isALPHA(c) || isDIGIT(c)) +#endif + +#ifndef isASCII +# define isASCII(c) ((c) <= 127) +#endif + +#ifndef isCNTRL +# define isCNTRL(c) ((c) < ' ' || (c) == 127) +#endif + +#ifndef isGRAPH +# define isGRAPH(c) (isALNUM(c) || isPUNCT(c)) +#endif + +#ifndef isPRINT +# define isPRINT(c) (((c) >= 32 && (c) < 127)) +#endif + +#ifndef isPUNCT +# define isPUNCT(c) (((c) >= 33 && (c) <= 47) || ((c) >= 58 && (c) <= 64) || ((c) >= 91 && (c) <= 96) || ((c) >= 123 && (c) <= 126)) +#endif + +#ifndef isXDIGIT +# define isXDIGIT(c) (isDIGIT(c) || ((c) >= 'a' && (c) <= 'f') || ((c) >= 'A' && (c) <= 'F')) +#endif + +#endif + +#ifndef PERL_SIGNALS_UNSAFE_FLAG + +#define PERL_SIGNALS_UNSAFE_FLAG 0x0001 + +#if (PERL_BCDVERSION < 0x5008000) +# define D_PPP_PERL_SIGNALS_INIT PERL_SIGNALS_UNSAFE_FLAG +#else +# define D_PPP_PERL_SIGNALS_INIT 0 +#endif + +#if defined(NEED_PL_signals) +static U32 DPPP_(my_PL_signals) = D_PPP_PERL_SIGNALS_INIT; +#elif defined(NEED_PL_signals_GLOBAL) +U32 DPPP_(my_PL_signals) = D_PPP_PERL_SIGNALS_INIT; +#else +extern U32 DPPP_(my_PL_signals); +#endif +#define PL_signals DPPP_(my_PL_signals) + +#endif + +/* Hint: PL_ppaddr + * Calling an op via PL_ppaddr requires passing a context argument + * for threaded builds. Since the context argument is different for + * 5.005 perls, you can use aTHXR (supplied by ppport.h), which will + * automatically be defined as the correct argument. + */ + +#if (PERL_BCDVERSION <= 0x5005005) +/* Replace: 1 */ +# define PL_ppaddr ppaddr +# define PL_no_modify no_modify +/* Replace: 0 */ +#endif + +#if (PERL_BCDVERSION <= 0x5004005) +/* Replace: 1 */ +# define PL_DBsignal DBsignal +# define PL_DBsingle DBsingle +# define PL_DBsub DBsub +# define PL_DBtrace DBtrace +# define PL_Sv Sv +# define PL_bufend bufend +# define PL_bufptr bufptr +# define PL_compiling compiling +# define PL_copline copline +# define PL_curcop curcop +# define PL_curstash curstash +# define PL_debstash debstash +# define PL_defgv defgv +# define PL_diehook diehook +# define PL_dirty dirty +# define PL_dowarn dowarn +# define PL_errgv errgv +# define PL_expect expect +# define PL_hexdigit hexdigit +# define PL_hints hints +# define PL_laststatval laststatval +# define PL_lex_state lex_state +# define PL_lex_stuff lex_stuff +# define PL_linestr linestr +# define PL_na na +# define PL_perl_destruct_level perl_destruct_level +# define PL_perldb perldb +# define PL_rsfp_filters rsfp_filters +# define PL_rsfp rsfp +# define PL_stack_base stack_base +# define PL_stack_sp stack_sp +# define PL_statcache statcache +# define PL_stdingv stdingv +# define PL_sv_arenaroot sv_arenaroot +# define PL_sv_no sv_no +# define PL_sv_undef sv_undef +# define PL_sv_yes sv_yes +# define PL_tainted tainted +# define PL_tainting tainting +# define PL_tokenbuf tokenbuf +/* Replace: 0 */ +#endif + +/* Warning: PL_parser + * For perl versions earlier than 5.9.5, this is an always + * non-NULL dummy. Also, it cannot be dereferenced. Don't + * use it if you can avoid is and unless you absolutely know + * what you're doing. + * If you always check that PL_parser is non-NULL, you can + * define DPPP_PL_parser_NO_DUMMY to avoid the creation of + * a dummy parser structure. + */ + +#if (PERL_BCDVERSION >= 0x5009005) +# ifdef DPPP_PL_parser_NO_DUMMY +# define D_PPP_my_PL_parser_var(var) ((PL_parser ? PL_parser : \ + (croak("panic: PL_parser == NULL in %s:%d", \ + __FILE__, __LINE__), (yy_parser *) NULL))->var) +# else +# ifdef DPPP_PL_parser_NO_DUMMY_WARNING +# define D_PPP_parser_dummy_warning(var) +# else +# define D_PPP_parser_dummy_warning(var) \ + warn("warning: dummy PL_" #var " used in %s:%d", __FILE__, __LINE__), +# endif +# define D_PPP_my_PL_parser_var(var) ((PL_parser ? PL_parser : \ + (D_PPP_parser_dummy_warning(var) &DPPP_(dummy_PL_parser)))->var) +#if defined(NEED_PL_parser) +static yy_parser DPPP_(dummy_PL_parser); +#elif defined(NEED_PL_parser_GLOBAL) +yy_parser DPPP_(dummy_PL_parser); +#else +extern yy_parser DPPP_(dummy_PL_parser); +#endif + +# endif + +/* PL_expect, PL_copline, PL_rsfp, PL_rsfp_filters, PL_linestr, PL_bufptr, PL_bufend, PL_lex_state, PL_lex_stuff, PL_tokenbuf depends on PL_parser */ +/* Warning: PL_expect, PL_copline, PL_rsfp, PL_rsfp_filters, PL_linestr, PL_bufptr, PL_bufend, PL_lex_state, PL_lex_stuff, PL_tokenbuf + * Do not use this variable unless you know exactly what you're + * doint. It is internal to the perl parser and may change or even + * be removed in the future. As of perl 5.9.5, you have to check + * for (PL_parser != NULL) for this variable to have any effect. + * An always non-NULL PL_parser dummy is provided for earlier + * perl versions. + * If PL_parser is NULL when you try to access this variable, a + * dummy is being accessed instead and a warning is issued unless + * you define DPPP_PL_parser_NO_DUMMY_WARNING. + * If DPPP_PL_parser_NO_DUMMY is defined, the code trying to access + * this variable will croak with a panic message. + */ + +# define PL_expect D_PPP_my_PL_parser_var(expect) +# define PL_copline D_PPP_my_PL_parser_var(copline) +# define PL_rsfp D_PPP_my_PL_parser_var(rsfp) +# define PL_rsfp_filters D_PPP_my_PL_parser_var(rsfp_filters) +# define PL_linestr D_PPP_my_PL_parser_var(linestr) +# define PL_bufptr D_PPP_my_PL_parser_var(bufptr) +# define PL_bufend D_PPP_my_PL_parser_var(bufend) +# define PL_lex_state D_PPP_my_PL_parser_var(lex_state) +# define PL_lex_stuff D_PPP_my_PL_parser_var(lex_stuff) +# define PL_tokenbuf D_PPP_my_PL_parser_var(tokenbuf) + +#else + +/* ensure that PL_parser != NULL and cannot be dereferenced */ +# define PL_parser ((void *) 1) + +#endif +#ifndef mPUSHs +# define mPUSHs(s) PUSHs(sv_2mortal(s)) +#endif + +#ifndef PUSHmortal +# define PUSHmortal PUSHs(sv_newmortal()) +#endif + +#ifndef mPUSHp +# define mPUSHp(p,l) sv_setpvn(PUSHmortal, (p), (l)) +#endif + +#ifndef mPUSHn +# define mPUSHn(n) sv_setnv(PUSHmortal, (NV)(n)) +#endif + +#ifndef mPUSHi +# define mPUSHi(i) sv_setiv(PUSHmortal, (IV)(i)) +#endif + +#ifndef mPUSHu +# define mPUSHu(u) sv_setuv(PUSHmortal, (UV)(u)) +#endif +#ifndef mXPUSHs +# define mXPUSHs(s) XPUSHs(sv_2mortal(s)) +#endif + +#ifndef XPUSHmortal +# define XPUSHmortal XPUSHs(sv_newmortal()) +#endif + +#ifndef mXPUSHp +# define mXPUSHp(p,l) STMT_START { EXTEND(sp,1); sv_setpvn(PUSHmortal, (p), (l)); } STMT_END +#endif + +#ifndef mXPUSHn +# define mXPUSHn(n) STMT_START { EXTEND(sp,1); sv_setnv(PUSHmortal, (NV)(n)); } STMT_END +#endif + +#ifndef mXPUSHi +# define mXPUSHi(i) STMT_START { EXTEND(sp,1); sv_setiv(PUSHmortal, (IV)(i)); } STMT_END +#endif + +#ifndef mXPUSHu +# define mXPUSHu(u) STMT_START { EXTEND(sp,1); sv_setuv(PUSHmortal, (UV)(u)); } STMT_END +#endif + +/* Replace: 1 */ +#ifndef call_sv +# define call_sv perl_call_sv +#endif + +#ifndef call_pv +# define call_pv perl_call_pv +#endif + +#ifndef call_argv +# define call_argv perl_call_argv +#endif + +#ifndef call_method +# define call_method perl_call_method +#endif +#ifndef eval_sv +# define eval_sv perl_eval_sv +#endif + +/* Replace: 0 */ +#ifndef PERL_LOADMOD_DENY +# define PERL_LOADMOD_DENY 0x1 +#endif + +#ifndef PERL_LOADMOD_NOIMPORT +# define PERL_LOADMOD_NOIMPORT 0x2 +#endif + +#ifndef PERL_LOADMOD_IMPORT_OPS +# define PERL_LOADMOD_IMPORT_OPS 0x4 +#endif + +#ifndef G_METHOD +# define G_METHOD 64 +# ifdef call_sv +# undef call_sv +# endif +# if (PERL_BCDVERSION < 0x5006000) +# define call_sv(sv, flags) ((flags) & G_METHOD ? perl_call_method((char *) SvPV_nolen_const(sv), \ + (flags) & ~G_METHOD) : perl_call_sv(sv, flags)) +# else +# define call_sv(sv, flags) ((flags) & G_METHOD ? Perl_call_method(aTHX_ (char *) SvPV_nolen_const(sv), \ + (flags) & ~G_METHOD) : Perl_call_sv(aTHX_ sv, flags)) +# endif +#endif + +/* Replace perl_eval_pv with eval_pv */ + +#ifndef eval_pv +#if defined(NEED_eval_pv) +static SV* DPPP_(my_eval_pv)(char *p, I32 croak_on_error); +static +#else +extern SV* DPPP_(my_eval_pv)(char *p, I32 croak_on_error); +#endif + +#ifdef eval_pv +# undef eval_pv +#endif +#define eval_pv(a,b) DPPP_(my_eval_pv)(aTHX_ a,b) +#define Perl_eval_pv DPPP_(my_eval_pv) + +#if defined(NEED_eval_pv) || defined(NEED_eval_pv_GLOBAL) + +SV* +DPPP_(my_eval_pv)(char *p, I32 croak_on_error) +{ + dSP; + SV* sv = newSVpv(p, 0); + + PUSHMARK(sp); + eval_sv(sv, G_SCALAR); + SvREFCNT_dec(sv); + + SPAGAIN; + sv = POPs; + PUTBACK; + + if (croak_on_error && SvTRUE(GvSV(errgv))) + croak(SvPVx(GvSV(errgv), na)); + + return sv; +} + +#endif +#endif + +#ifndef vload_module +#if defined(NEED_vload_module) +static void DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args); +static +#else +extern void DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args); +#endif + +#ifdef vload_module +# undef vload_module +#endif +#define vload_module(a,b,c,d) DPPP_(my_vload_module)(aTHX_ a,b,c,d) +#define Perl_vload_module DPPP_(my_vload_module) + +#if defined(NEED_vload_module) || defined(NEED_vload_module_GLOBAL) + +void +DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args) +{ + dTHR; + dVAR; + OP *veop, *imop; + + OP * const modname = newSVOP(OP_CONST, 0, name); + /* 5.005 has a somewhat hacky force_normal that doesn't croak on + SvREADONLY() if PL_compling is true. Current perls take care in + ck_require() to correctly turn off SvREADONLY before calling + force_normal_flags(). This seems a better fix than fudging PL_compling + */ + SvREADONLY_off(((SVOP*)modname)->op_sv); + modname->op_private |= OPpCONST_BARE; + if (ver) { + veop = newSVOP(OP_CONST, 0, ver); + } + else + veop = NULL; + if (flags & PERL_LOADMOD_NOIMPORT) { + imop = sawparens(newNULLLIST()); + } + else if (flags & PERL_LOADMOD_IMPORT_OPS) { + imop = va_arg(*args, OP*); + } + else { + SV *sv; + imop = NULL; + sv = va_arg(*args, SV*); + while (sv) { + imop = append_elem(OP_LIST, imop, newSVOP(OP_CONST, 0, sv)); + sv = va_arg(*args, SV*); + } + } + { + const line_t ocopline = PL_copline; + COP * const ocurcop = PL_curcop; + const int oexpect = PL_expect; + +#if (PERL_BCDVERSION >= 0x5004000) + utilize(!(flags & PERL_LOADMOD_DENY), start_subparse(FALSE, 0), + veop, modname, imop); +#else + utilize(!(flags & PERL_LOADMOD_DENY), start_subparse(), + modname, imop); +#endif + PL_expect = oexpect; + PL_copline = ocopline; + PL_curcop = ocurcop; + } +} + +#endif +#endif + +#ifndef load_module +#if defined(NEED_load_module) +static void DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...); +static +#else +extern void DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...); +#endif + +#ifdef load_module +# undef load_module +#endif +#define load_module DPPP_(my_load_module) +#define Perl_load_module DPPP_(my_load_module) + +#if defined(NEED_load_module) || defined(NEED_load_module_GLOBAL) + +void +DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...) +{ + va_list args; + va_start(args, ver); + vload_module(flags, name, ver, &args); + va_end(args); +} + +#endif +#endif +#ifndef newRV_inc +# define newRV_inc(sv) newRV(sv) /* Replace */ +#endif + +#ifndef newRV_noinc +#if defined(NEED_newRV_noinc) +static SV * DPPP_(my_newRV_noinc)(SV *sv); +static +#else +extern SV * DPPP_(my_newRV_noinc)(SV *sv); +#endif + +#ifdef newRV_noinc +# undef newRV_noinc +#endif +#define newRV_noinc(a) DPPP_(my_newRV_noinc)(aTHX_ a) +#define Perl_newRV_noinc DPPP_(my_newRV_noinc) + +#if defined(NEED_newRV_noinc) || defined(NEED_newRV_noinc_GLOBAL) +SV * +DPPP_(my_newRV_noinc)(SV *sv) +{ + SV *rv = (SV *)newRV(sv); + SvREFCNT_dec(sv); + return rv; +} +#endif +#endif + +/* Hint: newCONSTSUB + * Returns a CV* as of perl-5.7.1. This return value is not supported + * by Devel::PPPort. + */ + +/* newCONSTSUB from IO.xs is in the core starting with 5.004_63 */ +#if (PERL_BCDVERSION < 0x5004063) && (PERL_BCDVERSION != 0x5004005) +#if defined(NEED_newCONSTSUB) +static void DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv); +static +#else +extern void DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv); +#endif + +#ifdef newCONSTSUB +# undef newCONSTSUB +#endif +#define newCONSTSUB(a,b,c) DPPP_(my_newCONSTSUB)(aTHX_ a,b,c) +#define Perl_newCONSTSUB DPPP_(my_newCONSTSUB) + +#if defined(NEED_newCONSTSUB) || defined(NEED_newCONSTSUB_GLOBAL) + +/* This is just a trick to avoid a dependency of newCONSTSUB on PL_parser */ +/* (There's no PL_parser in perl < 5.005, so this is completely safe) */ +#define D_PPP_PL_copline PL_copline + +void +DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv) +{ + U32 oldhints = PL_hints; + HV *old_cop_stash = PL_curcop->cop_stash; + HV *old_curstash = PL_curstash; + line_t oldline = PL_curcop->cop_line; + PL_curcop->cop_line = D_PPP_PL_copline; + + PL_hints &= ~HINT_BLOCK_SCOPE; + if (stash) + PL_curstash = PL_curcop->cop_stash = stash; + + newSUB( + +#if (PERL_BCDVERSION < 0x5003022) + start_subparse(), +#elif (PERL_BCDVERSION == 0x5003022) + start_subparse(0), +#else /* 5.003_23 onwards */ + start_subparse(FALSE, 0), +#endif + + newSVOP(OP_CONST, 0, newSVpv((char *) name, 0)), + newSVOP(OP_CONST, 0, &PL_sv_no), /* SvPV(&PL_sv_no) == "" -- GMB */ + newSTATEOP(0, Nullch, newSVOP(OP_CONST, 0, sv)) + ); + + PL_hints = oldhints; + PL_curcop->cop_stash = old_cop_stash; + PL_curstash = old_curstash; + PL_curcop->cop_line = oldline; +} +#endif +#endif + +/* + * Boilerplate macros for initializing and accessing interpreter-local + * data from C. All statics in extensions should be reworked to use + * this, if you want to make the extension thread-safe. See ext/re/re.xs + * for an example of the use of these macros. + * + * Code that uses these macros is responsible for the following: + * 1. #define MY_CXT_KEY to a unique string, e.g. "DynaLoader_guts" + * 2. Declare a typedef named my_cxt_t that is a structure that contains + * all the data that needs to be interpreter-local. + * 3. Use the START_MY_CXT macro after the declaration of my_cxt_t. + * 4. Use the MY_CXT_INIT macro such that it is called exactly once + * (typically put in the BOOT: section). + * 5. Use the members of the my_cxt_t structure everywhere as + * MY_CXT.member. + * 6. Use the dMY_CXT macro (a declaration) in all the functions that + * access MY_CXT. + */ + +#if defined(MULTIPLICITY) || defined(PERL_OBJECT) || \ + defined(PERL_CAPI) || defined(PERL_IMPLICIT_CONTEXT) + +#ifndef START_MY_CXT + +/* This must appear in all extensions that define a my_cxt_t structure, + * right after the definition (i.e. at file scope). The non-threads + * case below uses it to declare the data as static. */ +#define START_MY_CXT + +#if (PERL_BCDVERSION < 0x5004068) +/* Fetches the SV that keeps the per-interpreter data. */ +#define dMY_CXT_SV \ + SV *my_cxt_sv = get_sv(MY_CXT_KEY, FALSE) +#else /* >= perl5.004_68 */ +#define dMY_CXT_SV \ + SV *my_cxt_sv = *hv_fetch(PL_modglobal, MY_CXT_KEY, \ + sizeof(MY_CXT_KEY)-1, TRUE) +#endif /* < perl5.004_68 */ + +/* This declaration should be used within all functions that use the + * interpreter-local data. */ +#define dMY_CXT \ + dMY_CXT_SV; \ + my_cxt_t *my_cxtp = INT2PTR(my_cxt_t*,SvUV(my_cxt_sv)) + +/* Creates and zeroes the per-interpreter data. + * (We allocate my_cxtp in a Perl SV so that it will be released when + * the interpreter goes away.) */ +#define MY_CXT_INIT \ + dMY_CXT_SV; \ + /* newSV() allocates one more than needed */ \ + my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ + Zero(my_cxtp, 1, my_cxt_t); \ + sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) + +/* This macro must be used to access members of the my_cxt_t structure. + * e.g. MYCXT.some_data */ +#define MY_CXT (*my_cxtp) + +/* Judicious use of these macros can reduce the number of times dMY_CXT + * is used. Use is similar to pTHX, aTHX etc. */ +#define pMY_CXT my_cxt_t *my_cxtp +#define pMY_CXT_ pMY_CXT, +#define _pMY_CXT ,pMY_CXT +#define aMY_CXT my_cxtp +#define aMY_CXT_ aMY_CXT, +#define _aMY_CXT ,aMY_CXT + +#endif /* START_MY_CXT */ + +#ifndef MY_CXT_CLONE +/* Clones the per-interpreter data. */ +#define MY_CXT_CLONE \ + dMY_CXT_SV; \ + my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ + Copy(INT2PTR(my_cxt_t*, SvUV(my_cxt_sv)), my_cxtp, 1, my_cxt_t);\ + sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) +#endif + +#else /* single interpreter */ + +#ifndef START_MY_CXT + +#define START_MY_CXT static my_cxt_t my_cxt; +#define dMY_CXT_SV dNOOP +#define dMY_CXT dNOOP +#define MY_CXT_INIT NOOP +#define MY_CXT my_cxt + +#define pMY_CXT void +#define pMY_CXT_ +#define _pMY_CXT +#define aMY_CXT +#define aMY_CXT_ +#define _aMY_CXT + +#endif /* START_MY_CXT */ + +#ifndef MY_CXT_CLONE +#define MY_CXT_CLONE NOOP +#endif + +#endif + +#ifndef IVdf +# if IVSIZE == LONGSIZE +# define IVdf "ld" +# define UVuf "lu" +# define UVof "lo" +# define UVxf "lx" +# define UVXf "lX" +# else +# if IVSIZE == INTSIZE +# define IVdf "d" +# define UVuf "u" +# define UVof "o" +# define UVxf "x" +# define UVXf "X" +# endif +# endif +#endif + +#ifndef NVef +# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) && \ + defined(PERL_PRIfldbl) && (PERL_BCDVERSION != 0x5006000) + /* Not very likely, but let's try anyway. */ +# define NVef PERL_PRIeldbl +# define NVff PERL_PRIfldbl +# define NVgf PERL_PRIgldbl +# else +# define NVef "e" +# define NVff "f" +# define NVgf "g" +# endif +#endif + +#ifndef SvREFCNT_inc +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc(sv) \ + ({ \ + SV * const _sv = (SV*)(sv); \ + if (_sv) \ + (SvREFCNT(_sv))++; \ + _sv; \ + }) +# else +# define SvREFCNT_inc(sv) \ + ((PL_Sv=(SV*)(sv)) ? (++(SvREFCNT(PL_Sv)),PL_Sv) : NULL) +# endif +#endif + +#ifndef SvREFCNT_inc_simple +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc_simple(sv) \ + ({ \ + if (sv) \ + (SvREFCNT(sv))++; \ + (SV *)(sv); \ + }) +# else +# define SvREFCNT_inc_simple(sv) \ + ((sv) ? (SvREFCNT(sv)++,(SV*)(sv)) : NULL) +# endif +#endif + +#ifndef SvREFCNT_inc_NN +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc_NN(sv) \ + ({ \ + SV * const _sv = (SV*)(sv); \ + SvREFCNT(_sv)++; \ + _sv; \ + }) +# else +# define SvREFCNT_inc_NN(sv) \ + (PL_Sv=(SV*)(sv),++(SvREFCNT(PL_Sv)),PL_Sv) +# endif +#endif + +#ifndef SvREFCNT_inc_void +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc_void(sv) \ + ({ \ + SV * const _sv = (SV*)(sv); \ + if (_sv) \ + (void)(SvREFCNT(_sv)++); \ + }) +# else +# define SvREFCNT_inc_void(sv) \ + (void)((PL_Sv=(SV*)(sv)) ? ++(SvREFCNT(PL_Sv)) : 0) +# endif +#endif +#ifndef SvREFCNT_inc_simple_void +# define SvREFCNT_inc_simple_void(sv) STMT_START { if (sv) SvREFCNT(sv)++; } STMT_END +#endif + +#ifndef SvREFCNT_inc_simple_NN +# define SvREFCNT_inc_simple_NN(sv) (++SvREFCNT(sv), (SV*)(sv)) +#endif + +#ifndef SvREFCNT_inc_void_NN +# define SvREFCNT_inc_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) +#endif + +#ifndef SvREFCNT_inc_simple_void_NN +# define SvREFCNT_inc_simple_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) +#endif + +#if (PERL_BCDVERSION < 0x5006000) +# define D_PPP_CONSTPV_ARG(x) ((char *) (x)) +#else +# define D_PPP_CONSTPV_ARG(x) (x) +#endif +#ifndef newSVpvn +# define newSVpvn(data,len) ((data) \ + ? ((len) ? newSVpv((data), (len)) : newSVpv("", 0)) \ + : newSV(0)) +#endif +#ifndef newSVpvn_utf8 +# define newSVpvn_utf8(s, len, u) newSVpvn_flags((s), (len), (u) ? SVf_UTF8 : 0) +#endif +#ifndef SVf_UTF8 +# define SVf_UTF8 0 +#endif + +#ifndef newSVpvn_flags + +#if defined(NEED_newSVpvn_flags) +static SV * DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags); +static +#else +extern SV * DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags); +#endif + +#ifdef newSVpvn_flags +# undef newSVpvn_flags +#endif +#define newSVpvn_flags(a,b,c) DPPP_(my_newSVpvn_flags)(aTHX_ a,b,c) +#define Perl_newSVpvn_flags DPPP_(my_newSVpvn_flags) + +#if defined(NEED_newSVpvn_flags) || defined(NEED_newSVpvn_flags_GLOBAL) + +SV * +DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags) +{ + SV *sv = newSVpvn(D_PPP_CONSTPV_ARG(s), len); + SvFLAGS(sv) |= (flags & SVf_UTF8); + return (flags & SVs_TEMP) ? sv_2mortal(sv) : sv; +} + +#endif + +#endif + +/* Backwards compatibility stuff... :-( */ +#if !defined(NEED_sv_2pv_flags) && defined(NEED_sv_2pv_nolen) +# define NEED_sv_2pv_flags +#endif +#if !defined(NEED_sv_2pv_flags_GLOBAL) && defined(NEED_sv_2pv_nolen_GLOBAL) +# define NEED_sv_2pv_flags_GLOBAL +#endif + +/* Hint: sv_2pv_nolen + * Use the SvPV_nolen() or SvPV_nolen_const() macros instead of sv_2pv_nolen(). + */ +#ifndef sv_2pv_nolen +# define sv_2pv_nolen(sv) SvPV_nolen(sv) +#endif + +#ifdef SvPVbyte + +/* Hint: SvPVbyte + * Does not work in perl-5.6.1, ppport.h implements a version + * borrowed from perl-5.7.3. + */ + +#if (PERL_BCDVERSION < 0x5007000) + +#if defined(NEED_sv_2pvbyte) +static char * DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp); +static +#else +extern char * DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp); +#endif + +#ifdef sv_2pvbyte +# undef sv_2pvbyte +#endif +#define sv_2pvbyte(a,b) DPPP_(my_sv_2pvbyte)(aTHX_ a,b) +#define Perl_sv_2pvbyte DPPP_(my_sv_2pvbyte) + +#if defined(NEED_sv_2pvbyte) || defined(NEED_sv_2pvbyte_GLOBAL) + +char * +DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp) +{ + sv_utf8_downgrade(sv,0); + return SvPV(sv,*lp); +} + +#endif + +/* Hint: sv_2pvbyte + * Use the SvPVbyte() macro instead of sv_2pvbyte(). + */ + +#undef SvPVbyte + +#define SvPVbyte(sv, lp) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_UTF8)) == (SVf_POK) \ + ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_2pvbyte(sv, &lp)) + +#endif + +#else + +# define SvPVbyte SvPV +# define sv_2pvbyte sv_2pv + +#endif +#ifndef sv_2pvbyte_nolen +# define sv_2pvbyte_nolen(sv) sv_2pv_nolen(sv) +#endif + +/* Hint: sv_pvn + * Always use the SvPV() macro instead of sv_pvn(). + */ + +/* Hint: sv_pvn_force + * Always use the SvPV_force() macro instead of sv_pvn_force(). + */ + +/* If these are undefined, they're not handled by the core anyway */ +#ifndef SV_IMMEDIATE_UNREF +# define SV_IMMEDIATE_UNREF 0 +#endif + +#ifndef SV_GMAGIC +# define SV_GMAGIC 0 +#endif + +#ifndef SV_COW_DROP_PV +# define SV_COW_DROP_PV 0 +#endif + +#ifndef SV_UTF8_NO_ENCODING +# define SV_UTF8_NO_ENCODING 0 +#endif + +#ifndef SV_NOSTEAL +# define SV_NOSTEAL 0 +#endif + +#ifndef SV_CONST_RETURN +# define SV_CONST_RETURN 0 +#endif + +#ifndef SV_MUTABLE_RETURN +# define SV_MUTABLE_RETURN 0 +#endif + +#ifndef SV_SMAGIC +# define SV_SMAGIC 0 +#endif + +#ifndef SV_HAS_TRAILING_NUL +# define SV_HAS_TRAILING_NUL 0 +#endif + +#ifndef SV_COW_SHARED_HASH_KEYS +# define SV_COW_SHARED_HASH_KEYS 0 +#endif + +#if (PERL_BCDVERSION < 0x5007002) + +#if defined(NEED_sv_2pv_flags) +static char * DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +static +#else +extern char * DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +#endif + +#ifdef sv_2pv_flags +# undef sv_2pv_flags +#endif +#define sv_2pv_flags(a,b,c) DPPP_(my_sv_2pv_flags)(aTHX_ a,b,c) +#define Perl_sv_2pv_flags DPPP_(my_sv_2pv_flags) + +#if defined(NEED_sv_2pv_flags) || defined(NEED_sv_2pv_flags_GLOBAL) + +char * +DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags) +{ + STRLEN n_a = (STRLEN) flags; + return sv_2pv(sv, lp ? lp : &n_a); +} + +#endif + +#if defined(NEED_sv_pvn_force_flags) +static char * DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +static +#else +extern char * DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +#endif + +#ifdef sv_pvn_force_flags +# undef sv_pvn_force_flags +#endif +#define sv_pvn_force_flags(a,b,c) DPPP_(my_sv_pvn_force_flags)(aTHX_ a,b,c) +#define Perl_sv_pvn_force_flags DPPP_(my_sv_pvn_force_flags) + +#if defined(NEED_sv_pvn_force_flags) || defined(NEED_sv_pvn_force_flags_GLOBAL) + +char * +DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags) +{ + STRLEN n_a = (STRLEN) flags; + return sv_pvn_force(sv, lp ? lp : &n_a); +} + +#endif + +#endif + +#if (PERL_BCDVERSION < 0x5008008) || ( (PERL_BCDVERSION >= 0x5009000) && (PERL_BCDVERSION < 0x5009003) ) +# define DPPP_SVPV_NOLEN_LP_ARG &PL_na +#else +# define DPPP_SVPV_NOLEN_LP_ARG 0 +#endif +#ifndef SvPV_const +# define SvPV_const(sv, lp) SvPV_flags_const(sv, lp, SV_GMAGIC) +#endif + +#ifndef SvPV_mutable +# define SvPV_mutable(sv, lp) SvPV_flags_mutable(sv, lp, SV_GMAGIC) +#endif +#ifndef SvPV_flags +# define SvPV_flags(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_2pv_flags(sv, &lp, flags)) +#endif +#ifndef SvPV_flags_const +# define SvPV_flags_const(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX_const(sv)) : \ + (const char*) sv_2pv_flags(sv, &lp, flags|SV_CONST_RETURN)) +#endif +#ifndef SvPV_flags_const_nolen +# define SvPV_flags_const_nolen(sv, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? SvPVX_const(sv) : \ + (const char*) sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, flags|SV_CONST_RETURN)) +#endif +#ifndef SvPV_flags_mutable +# define SvPV_flags_mutable(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX_mutable(sv)) : \ + sv_2pv_flags(sv, &lp, flags|SV_MUTABLE_RETURN)) +#endif +#ifndef SvPV_force +# define SvPV_force(sv, lp) SvPV_force_flags(sv, lp, SV_GMAGIC) +#endif + +#ifndef SvPV_force_nolen +# define SvPV_force_nolen(sv) SvPV_force_flags_nolen(sv, SV_GMAGIC) +#endif + +#ifndef SvPV_force_mutable +# define SvPV_force_mutable(sv, lp) SvPV_force_flags_mutable(sv, lp, SV_GMAGIC) +#endif + +#ifndef SvPV_force_nomg +# define SvPV_force_nomg(sv, lp) SvPV_force_flags(sv, lp, 0) +#endif + +#ifndef SvPV_force_nomg_nolen +# define SvPV_force_nomg_nolen(sv) SvPV_force_flags_nolen(sv, 0) +#endif +#ifndef SvPV_force_flags +# define SvPV_force_flags(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_pvn_force_flags(sv, &lp, flags)) +#endif +#ifndef SvPV_force_flags_nolen +# define SvPV_force_flags_nolen(sv, flags) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ + ? SvPVX(sv) : sv_pvn_force_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, flags)) +#endif +#ifndef SvPV_force_flags_mutable +# define SvPV_force_flags_mutable(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX_mutable(sv)) \ + : sv_pvn_force_flags(sv, &lp, flags|SV_MUTABLE_RETURN)) +#endif +#ifndef SvPV_nolen +# define SvPV_nolen(sv) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? SvPVX(sv) : sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, SV_GMAGIC)) +#endif +#ifndef SvPV_nolen_const +# define SvPV_nolen_const(sv) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? SvPVX_const(sv) : sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, SV_GMAGIC|SV_CONST_RETURN)) +#endif +#ifndef SvPV_nomg +# define SvPV_nomg(sv, lp) SvPV_flags(sv, lp, 0) +#endif + +#ifndef SvPV_nomg_const +# define SvPV_nomg_const(sv, lp) SvPV_flags_const(sv, lp, 0) +#endif + +#ifndef SvPV_nomg_const_nolen +# define SvPV_nomg_const_nolen(sv) SvPV_flags_const_nolen(sv, 0) +#endif +#ifndef SvPV_renew +# define SvPV_renew(sv,n) STMT_START { SvLEN_set(sv, n); \ + SvPV_set((sv), (char *) saferealloc( \ + (Malloc_t)SvPVX(sv), (MEM_SIZE)((n)))); \ + } STMT_END +#endif +#ifndef SvMAGIC_set +# define SvMAGIC_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_PVMG); \ + (((XPVMG*) SvANY(sv))->xmg_magic = (val)); } STMT_END +#endif + +#if (PERL_BCDVERSION < 0x5009003) +#ifndef SvPVX_const +# define SvPVX_const(sv) ((const char*) (0 + SvPVX(sv))) +#endif + +#ifndef SvPVX_mutable +# define SvPVX_mutable(sv) (0 + SvPVX(sv)) +#endif +#ifndef SvRV_set +# define SvRV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_RV); \ + (((XRV*) SvANY(sv))->xrv_rv = (val)); } STMT_END +#endif + +#else +#ifndef SvPVX_const +# define SvPVX_const(sv) ((const char*)((sv)->sv_u.svu_pv)) +#endif + +#ifndef SvPVX_mutable +# define SvPVX_mutable(sv) ((sv)->sv_u.svu_pv) +#endif +#ifndef SvRV_set +# define SvRV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_RV); \ + ((sv)->sv_u.svu_rv = (val)); } STMT_END +#endif + +#endif +#ifndef SvSTASH_set +# define SvSTASH_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_PVMG); \ + (((XPVMG*) SvANY(sv))->xmg_stash = (val)); } STMT_END +#endif + +#if (PERL_BCDVERSION < 0x5004000) +#ifndef SvUV_set +# define SvUV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) == SVt_IV || SvTYPE(sv) >= SVt_PVIV); \ + (((XPVIV*) SvANY(sv))->xiv_iv = (IV) (val)); } STMT_END +#endif + +#else +#ifndef SvUV_set +# define SvUV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) == SVt_IV || SvTYPE(sv) >= SVt_PVIV); \ + (((XPVUV*) SvANY(sv))->xuv_uv = (val)); } STMT_END +#endif + +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(vnewSVpvf) +#if defined(NEED_vnewSVpvf) +static SV * DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args); +static +#else +extern SV * DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args); +#endif + +#ifdef vnewSVpvf +# undef vnewSVpvf +#endif +#define vnewSVpvf(a,b) DPPP_(my_vnewSVpvf)(aTHX_ a,b) +#define Perl_vnewSVpvf DPPP_(my_vnewSVpvf) + +#if defined(NEED_vnewSVpvf) || defined(NEED_vnewSVpvf_GLOBAL) + +SV * +DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args) +{ + register SV *sv = newSV(0); + sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); + return sv; +} + +#endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vcatpvf) +# define sv_vcatpvf(sv, pat, args) sv_vcatpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)) +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vsetpvf) +# define sv_vsetpvf(sv, pat, args) sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)) +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_catpvf_mg) +#if defined(NEED_sv_catpvf_mg) +static void DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +#endif + +#define Perl_sv_catpvf_mg DPPP_(my_sv_catpvf_mg) + +#if defined(NEED_sv_catpvf_mg) || defined(NEED_sv_catpvf_mg_GLOBAL) + +void +DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...) +{ + va_list args; + va_start(args, pat); + sv_vcatpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif + +#ifdef PERL_IMPLICIT_CONTEXT +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_catpvf_mg_nocontext) +#if defined(NEED_sv_catpvf_mg_nocontext) +static void DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...); +#endif + +#define sv_catpvf_mg_nocontext DPPP_(my_sv_catpvf_mg_nocontext) +#define Perl_sv_catpvf_mg_nocontext DPPP_(my_sv_catpvf_mg_nocontext) + +#if defined(NEED_sv_catpvf_mg_nocontext) || defined(NEED_sv_catpvf_mg_nocontext_GLOBAL) + +void +DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...) +{ + dTHX; + va_list args; + va_start(args, pat); + sv_vcatpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif +#endif + +/* sv_catpvf_mg depends on sv_catpvf_mg_nocontext */ +#ifndef sv_catpvf_mg +# ifdef PERL_IMPLICIT_CONTEXT +# define sv_catpvf_mg Perl_sv_catpvf_mg_nocontext +# else +# define sv_catpvf_mg Perl_sv_catpvf_mg +# endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vcatpvf_mg) +# define sv_vcatpvf_mg(sv, pat, args) \ + STMT_START { \ + sv_vcatpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); \ + SvSETMAGIC(sv); \ + } STMT_END +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_setpvf_mg) +#if defined(NEED_sv_setpvf_mg) +static void DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +#endif + +#define Perl_sv_setpvf_mg DPPP_(my_sv_setpvf_mg) + +#if defined(NEED_sv_setpvf_mg) || defined(NEED_sv_setpvf_mg_GLOBAL) + +void +DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...) +{ + va_list args; + va_start(args, pat); + sv_vsetpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif + +#ifdef PERL_IMPLICIT_CONTEXT +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_setpvf_mg_nocontext) +#if defined(NEED_sv_setpvf_mg_nocontext) +static void DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...); +#endif + +#define sv_setpvf_mg_nocontext DPPP_(my_sv_setpvf_mg_nocontext) +#define Perl_sv_setpvf_mg_nocontext DPPP_(my_sv_setpvf_mg_nocontext) + +#if defined(NEED_sv_setpvf_mg_nocontext) || defined(NEED_sv_setpvf_mg_nocontext_GLOBAL) + +void +DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...) +{ + dTHX; + va_list args; + va_start(args, pat); + sv_vsetpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif +#endif + +/* sv_setpvf_mg depends on sv_setpvf_mg_nocontext */ +#ifndef sv_setpvf_mg +# ifdef PERL_IMPLICIT_CONTEXT +# define sv_setpvf_mg Perl_sv_setpvf_mg_nocontext +# else +# define sv_setpvf_mg Perl_sv_setpvf_mg +# endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vsetpvf_mg) +# define sv_vsetpvf_mg(sv, pat, args) \ + STMT_START { \ + sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); \ + SvSETMAGIC(sv); \ + } STMT_END +#endif + +#ifndef newSVpvn_share + +#if defined(NEED_newSVpvn_share) +static SV * DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash); +static +#else +extern SV * DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash); +#endif + +#ifdef newSVpvn_share +# undef newSVpvn_share +#endif +#define newSVpvn_share(a,b,c) DPPP_(my_newSVpvn_share)(aTHX_ a,b,c) +#define Perl_newSVpvn_share DPPP_(my_newSVpvn_share) + +#if defined(NEED_newSVpvn_share) || defined(NEED_newSVpvn_share_GLOBAL) + +SV * +DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash) +{ + SV *sv; + if (len < 0) + len = -len; + if (!hash) + PERL_HASH(hash, (char*) src, len); + sv = newSVpvn((char *) src, len); + sv_upgrade(sv, SVt_PVIV); + SvIVX(sv) = hash; + SvREADONLY_on(sv); + SvPOK_on(sv); + return sv; +} + +#endif + +#endif +#ifndef SvSHARED_HASH +# define SvSHARED_HASH(sv) (0 + SvUVX(sv)) +#endif +#ifndef WARN_ALL +# define WARN_ALL 0 +#endif + +#ifndef WARN_CLOSURE +# define WARN_CLOSURE 1 +#endif + +#ifndef WARN_DEPRECATED +# define WARN_DEPRECATED 2 +#endif + +#ifndef WARN_EXITING +# define WARN_EXITING 3 +#endif + +#ifndef WARN_GLOB +# define WARN_GLOB 4 +#endif + +#ifndef WARN_IO +# define WARN_IO 5 +#endif + +#ifndef WARN_CLOSED +# define WARN_CLOSED 6 +#endif + +#ifndef WARN_EXEC +# define WARN_EXEC 7 +#endif + +#ifndef WARN_LAYER +# define WARN_LAYER 8 +#endif + +#ifndef WARN_NEWLINE +# define WARN_NEWLINE 9 +#endif + +#ifndef WARN_PIPE +# define WARN_PIPE 10 +#endif + +#ifndef WARN_UNOPENED +# define WARN_UNOPENED 11 +#endif + +#ifndef WARN_MISC +# define WARN_MISC 12 +#endif + +#ifndef WARN_NUMERIC +# define WARN_NUMERIC 13 +#endif + +#ifndef WARN_ONCE +# define WARN_ONCE 14 +#endif + +#ifndef WARN_OVERFLOW +# define WARN_OVERFLOW 15 +#endif + +#ifndef WARN_PACK +# define WARN_PACK 16 +#endif + +#ifndef WARN_PORTABLE +# define WARN_PORTABLE 17 +#endif + +#ifndef WARN_RECURSION +# define WARN_RECURSION 18 +#endif + +#ifndef WARN_REDEFINE +# define WARN_REDEFINE 19 +#endif + +#ifndef WARN_REGEXP +# define WARN_REGEXP 20 +#endif + +#ifndef WARN_SEVERE +# define WARN_SEVERE 21 +#endif + +#ifndef WARN_DEBUGGING +# define WARN_DEBUGGING 22 +#endif + +#ifndef WARN_INPLACE +# define WARN_INPLACE 23 +#endif + +#ifndef WARN_INTERNAL +# define WARN_INTERNAL 24 +#endif + +#ifndef WARN_MALLOC +# define WARN_MALLOC 25 +#endif + +#ifndef WARN_SIGNAL +# define WARN_SIGNAL 26 +#endif + +#ifndef WARN_SUBSTR +# define WARN_SUBSTR 27 +#endif + +#ifndef WARN_SYNTAX +# define WARN_SYNTAX 28 +#endif + +#ifndef WARN_AMBIGUOUS +# define WARN_AMBIGUOUS 29 +#endif + +#ifndef WARN_BAREWORD +# define WARN_BAREWORD 30 +#endif + +#ifndef WARN_DIGIT +# define WARN_DIGIT 31 +#endif + +#ifndef WARN_PARENTHESIS +# define WARN_PARENTHESIS 32 +#endif + +#ifndef WARN_PRECEDENCE +# define WARN_PRECEDENCE 33 +#endif + +#ifndef WARN_PRINTF +# define WARN_PRINTF 34 +#endif + +#ifndef WARN_PROTOTYPE +# define WARN_PROTOTYPE 35 +#endif + +#ifndef WARN_QW +# define WARN_QW 36 +#endif + +#ifndef WARN_RESERVED +# define WARN_RESERVED 37 +#endif + +#ifndef WARN_SEMICOLON +# define WARN_SEMICOLON 38 +#endif + +#ifndef WARN_TAINT +# define WARN_TAINT 39 +#endif + +#ifndef WARN_THREADS +# define WARN_THREADS 40 +#endif + +#ifndef WARN_UNINITIALIZED +# define WARN_UNINITIALIZED 41 +#endif + +#ifndef WARN_UNPACK +# define WARN_UNPACK 42 +#endif + +#ifndef WARN_UNTIE +# define WARN_UNTIE 43 +#endif + +#ifndef WARN_UTF8 +# define WARN_UTF8 44 +#endif + +#ifndef WARN_VOID +# define WARN_VOID 45 +#endif + +#ifndef WARN_ASSERTIONS +# define WARN_ASSERTIONS 46 +#endif +#ifndef packWARN +# define packWARN(a) (a) +#endif + +#ifndef ckWARN +# ifdef G_WARN_ON +# define ckWARN(a) (PL_dowarn & G_WARN_ON) +# else +# define ckWARN(a) PL_dowarn +# endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(warner) +#if defined(NEED_warner) +static void DPPP_(my_warner)(U32 err, const char *pat, ...); +static +#else +extern void DPPP_(my_warner)(U32 err, const char *pat, ...); +#endif + +#define Perl_warner DPPP_(my_warner) + +#if defined(NEED_warner) || defined(NEED_warner_GLOBAL) + +void +DPPP_(my_warner)(U32 err, const char *pat, ...) +{ + SV *sv; + va_list args; + + PERL_UNUSED_ARG(err); + + va_start(args, pat); + sv = vnewSVpvf(pat, &args); + va_end(args); + sv_2mortal(sv); + warn("%s", SvPV_nolen(sv)); +} + +#define warner Perl_warner + +#define Perl_warner_nocontext Perl_warner + +#endif +#endif + +/* concatenating with "" ensures that only literal strings are accepted as argument + * note that STR_WITH_LEN() can't be used as argument to macros or functions that + * under some configurations might be macros + */ +#ifndef STR_WITH_LEN +# define STR_WITH_LEN(s) (s ""), (sizeof(s)-1) +#endif +#ifndef newSVpvs +# define newSVpvs(str) newSVpvn(str "", sizeof(str) - 1) +#endif + +#ifndef newSVpvs_flags +# define newSVpvs_flags(str, flags) newSVpvn_flags(str "", sizeof(str) - 1, flags) +#endif + +#ifndef sv_catpvs +# define sv_catpvs(sv, str) sv_catpvn(sv, str "", sizeof(str) - 1) +#endif + +#ifndef sv_setpvs +# define sv_setpvs(sv, str) sv_setpvn(sv, str "", sizeof(str) - 1) +#endif + +#ifndef hv_fetchs +# define hv_fetchs(hv, key, lval) hv_fetch(hv, key "", sizeof(key) - 1, lval) +#endif + +#ifndef hv_stores +# define hv_stores(hv, key, val) hv_store(hv, key "", sizeof(key) - 1, val, 0) +#endif +#ifndef SvGETMAGIC +# define SvGETMAGIC(x) STMT_START { if (SvGMAGICAL(x)) mg_get(x); } STMT_END +#endif +#ifndef PERL_MAGIC_sv +# define PERL_MAGIC_sv '\0' +#endif + +#ifndef PERL_MAGIC_overload +# define PERL_MAGIC_overload 'A' +#endif + +#ifndef PERL_MAGIC_overload_elem +# define PERL_MAGIC_overload_elem 'a' +#endif + +#ifndef PERL_MAGIC_overload_table +# define PERL_MAGIC_overload_table 'c' +#endif + +#ifndef PERL_MAGIC_bm +# define PERL_MAGIC_bm 'B' +#endif + +#ifndef PERL_MAGIC_regdata +# define PERL_MAGIC_regdata 'D' +#endif + +#ifndef PERL_MAGIC_regdatum +# define PERL_MAGIC_regdatum 'd' +#endif + +#ifndef PERL_MAGIC_env +# define PERL_MAGIC_env 'E' +#endif + +#ifndef PERL_MAGIC_envelem +# define PERL_MAGIC_envelem 'e' +#endif + +#ifndef PERL_MAGIC_fm +# define PERL_MAGIC_fm 'f' +#endif + +#ifndef PERL_MAGIC_regex_global +# define PERL_MAGIC_regex_global 'g' +#endif + +#ifndef PERL_MAGIC_isa +# define PERL_MAGIC_isa 'I' +#endif + +#ifndef PERL_MAGIC_isaelem +# define PERL_MAGIC_isaelem 'i' +#endif + +#ifndef PERL_MAGIC_nkeys +# define PERL_MAGIC_nkeys 'k' +#endif + +#ifndef PERL_MAGIC_dbfile +# define PERL_MAGIC_dbfile 'L' +#endif + +#ifndef PERL_MAGIC_dbline +# define PERL_MAGIC_dbline 'l' +#endif + +#ifndef PERL_MAGIC_mutex +# define PERL_MAGIC_mutex 'm' +#endif + +#ifndef PERL_MAGIC_shared +# define PERL_MAGIC_shared 'N' +#endif + +#ifndef PERL_MAGIC_shared_scalar +# define PERL_MAGIC_shared_scalar 'n' +#endif + +#ifndef PERL_MAGIC_collxfrm +# define PERL_MAGIC_collxfrm 'o' +#endif + +#ifndef PERL_MAGIC_tied +# define PERL_MAGIC_tied 'P' +#endif + +#ifndef PERL_MAGIC_tiedelem +# define PERL_MAGIC_tiedelem 'p' +#endif + +#ifndef PERL_MAGIC_tiedscalar +# define PERL_MAGIC_tiedscalar 'q' +#endif + +#ifndef PERL_MAGIC_qr +# define PERL_MAGIC_qr 'r' +#endif + +#ifndef PERL_MAGIC_sig +# define PERL_MAGIC_sig 'S' +#endif + +#ifndef PERL_MAGIC_sigelem +# define PERL_MAGIC_sigelem 's' +#endif + +#ifndef PERL_MAGIC_taint +# define PERL_MAGIC_taint 't' +#endif + +#ifndef PERL_MAGIC_uvar +# define PERL_MAGIC_uvar 'U' +#endif + +#ifndef PERL_MAGIC_uvar_elem +# define PERL_MAGIC_uvar_elem 'u' +#endif + +#ifndef PERL_MAGIC_vstring +# define PERL_MAGIC_vstring 'V' +#endif + +#ifndef PERL_MAGIC_vec +# define PERL_MAGIC_vec 'v' +#endif + +#ifndef PERL_MAGIC_utf8 +# define PERL_MAGIC_utf8 'w' +#endif + +#ifndef PERL_MAGIC_substr +# define PERL_MAGIC_substr 'x' +#endif + +#ifndef PERL_MAGIC_defelem +# define PERL_MAGIC_defelem 'y' +#endif + +#ifndef PERL_MAGIC_glob +# define PERL_MAGIC_glob '*' +#endif + +#ifndef PERL_MAGIC_arylen +# define PERL_MAGIC_arylen '#' +#endif + +#ifndef PERL_MAGIC_pos +# define PERL_MAGIC_pos '.' +#endif + +#ifndef PERL_MAGIC_backref +# define PERL_MAGIC_backref '<' +#endif + +#ifndef PERL_MAGIC_ext +# define PERL_MAGIC_ext '~' +#endif + +/* That's the best we can do... */ +#ifndef sv_catpvn_nomg +# define sv_catpvn_nomg sv_catpvn +#endif + +#ifndef sv_catsv_nomg +# define sv_catsv_nomg sv_catsv +#endif + +#ifndef sv_setsv_nomg +# define sv_setsv_nomg sv_setsv +#endif + +#ifndef sv_pvn_nomg +# define sv_pvn_nomg sv_pvn +#endif + +#ifndef SvIV_nomg +# define SvIV_nomg SvIV +#endif + +#ifndef SvUV_nomg +# define SvUV_nomg SvUV +#endif + +#ifndef sv_catpv_mg +# define sv_catpv_mg(sv, ptr) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_catpv(TeMpSv,ptr); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_catpvn_mg +# define sv_catpvn_mg(sv, ptr, len) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_catpvn(TeMpSv,ptr,len); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_catsv_mg +# define sv_catsv_mg(dsv, ssv) \ + STMT_START { \ + SV *TeMpSv = dsv; \ + sv_catsv(TeMpSv,ssv); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setiv_mg +# define sv_setiv_mg(sv, i) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setiv(TeMpSv,i); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setnv_mg +# define sv_setnv_mg(sv, num) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setnv(TeMpSv,num); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setpv_mg +# define sv_setpv_mg(sv, ptr) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setpv(TeMpSv,ptr); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setpvn_mg +# define sv_setpvn_mg(sv, ptr, len) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setpvn(TeMpSv,ptr,len); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setsv_mg +# define sv_setsv_mg(dsv, ssv) \ + STMT_START { \ + SV *TeMpSv = dsv; \ + sv_setsv(TeMpSv,ssv); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setuv_mg +# define sv_setuv_mg(sv, i) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setuv(TeMpSv,i); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_usepvn_mg +# define sv_usepvn_mg(sv, ptr, len) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_usepvn(TeMpSv,ptr,len); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif +#ifndef SvVSTRING_mg +# define SvVSTRING_mg(sv) (SvMAGICAL(sv) ? mg_find(sv, PERL_MAGIC_vstring) : NULL) +#endif + +/* Hint: sv_magic_portable + * This is a compatibility function that is only available with + * Devel::PPPort. It is NOT in the perl core. + * Its purpose is to mimic the 5.8.0 behaviour of sv_magic() when + * it is being passed a name pointer with namlen == 0. In that + * case, perl 5.8.0 and later store the pointer, not a copy of it. + * The compatibility can be provided back to perl 5.004. With + * earlier versions, the code will not compile. + */ + +#if (PERL_BCDVERSION < 0x5004000) + + /* code that uses sv_magic_portable will not compile */ + +#elif (PERL_BCDVERSION < 0x5008000) + +# define sv_magic_portable(sv, obj, how, name, namlen) \ + STMT_START { \ + SV *SvMp_sv = (sv); \ + char *SvMp_name = (char *) (name); \ + I32 SvMp_namlen = (namlen); \ + if (SvMp_name && SvMp_namlen == 0) \ + { \ + MAGIC *mg; \ + sv_magic(SvMp_sv, obj, how, 0, 0); \ + mg = SvMAGIC(SvMp_sv); \ + mg->mg_len = -42; /* XXX: this is the tricky part */ \ + mg->mg_ptr = SvMp_name; \ + } \ + else \ + { \ + sv_magic(SvMp_sv, obj, how, SvMp_name, SvMp_namlen); \ + } \ + } STMT_END + +#else + +# define sv_magic_portable(a, b, c, d, e) sv_magic(a, b, c, d, e) + +#endif + +#ifdef USE_ITHREADS +#ifndef CopFILE +# define CopFILE(c) ((c)->cop_file) +#endif + +#ifndef CopFILEGV +# define CopFILEGV(c) (CopFILE(c) ? gv_fetchfile(CopFILE(c)) : Nullgv) +#endif + +#ifndef CopFILE_set +# define CopFILE_set(c,pv) ((c)->cop_file = savepv(pv)) +#endif + +#ifndef CopFILESV +# define CopFILESV(c) (CopFILE(c) ? GvSV(gv_fetchfile(CopFILE(c))) : Nullsv) +#endif + +#ifndef CopFILEAV +# define CopFILEAV(c) (CopFILE(c) ? GvAV(gv_fetchfile(CopFILE(c))) : Nullav) +#endif + +#ifndef CopSTASHPV +# define CopSTASHPV(c) ((c)->cop_stashpv) +#endif + +#ifndef CopSTASHPV_set +# define CopSTASHPV_set(c,pv) ((c)->cop_stashpv = ((pv) ? savepv(pv) : Nullch)) +#endif + +#ifndef CopSTASH +# define CopSTASH(c) (CopSTASHPV(c) ? gv_stashpv(CopSTASHPV(c),GV_ADD) : Nullhv) +#endif + +#ifndef CopSTASH_set +# define CopSTASH_set(c,hv) CopSTASHPV_set(c, (hv) ? HvNAME(hv) : Nullch) +#endif + +#ifndef CopSTASH_eq +# define CopSTASH_eq(c,hv) ((hv) && (CopSTASHPV(c) == HvNAME(hv) \ + || (CopSTASHPV(c) && HvNAME(hv) \ + && strEQ(CopSTASHPV(c), HvNAME(hv))))) +#endif + +#else +#ifndef CopFILEGV +# define CopFILEGV(c) ((c)->cop_filegv) +#endif + +#ifndef CopFILEGV_set +# define CopFILEGV_set(c,gv) ((c)->cop_filegv = (GV*)SvREFCNT_inc(gv)) +#endif + +#ifndef CopFILE_set +# define CopFILE_set(c,pv) CopFILEGV_set((c), gv_fetchfile(pv)) +#endif + +#ifndef CopFILESV +# define CopFILESV(c) (CopFILEGV(c) ? GvSV(CopFILEGV(c)) : Nullsv) +#endif + +#ifndef CopFILEAV +# define CopFILEAV(c) (CopFILEGV(c) ? GvAV(CopFILEGV(c)) : Nullav) +#endif + +#ifndef CopFILE +# define CopFILE(c) (CopFILESV(c) ? SvPVX(CopFILESV(c)) : Nullch) +#endif + +#ifndef CopSTASH +# define CopSTASH(c) ((c)->cop_stash) +#endif + +#ifndef CopSTASH_set +# define CopSTASH_set(c,hv) ((c)->cop_stash = (hv)) +#endif + +#ifndef CopSTASHPV +# define CopSTASHPV(c) (CopSTASH(c) ? HvNAME(CopSTASH(c)) : Nullch) +#endif + +#ifndef CopSTASHPV_set +# define CopSTASHPV_set(c,pv) CopSTASH_set((c), gv_stashpv(pv,GV_ADD)) +#endif + +#ifndef CopSTASH_eq +# define CopSTASH_eq(c,hv) (CopSTASH(c) == (hv)) +#endif + +#endif /* USE_ITHREADS */ +#ifndef IN_PERL_COMPILETIME +# define IN_PERL_COMPILETIME (PL_curcop == &PL_compiling) +#endif + +#ifndef IN_LOCALE_RUNTIME +# define IN_LOCALE_RUNTIME (PL_curcop->op_private & HINT_LOCALE) +#endif + +#ifndef IN_LOCALE_COMPILETIME +# define IN_LOCALE_COMPILETIME (PL_hints & HINT_LOCALE) +#endif + +#ifndef IN_LOCALE +# define IN_LOCALE (IN_PERL_COMPILETIME ? IN_LOCALE_COMPILETIME : IN_LOCALE_RUNTIME) +#endif +#ifndef IS_NUMBER_IN_UV +# define IS_NUMBER_IN_UV 0x01 +#endif + +#ifndef IS_NUMBER_GREATER_THAN_UV_MAX +# define IS_NUMBER_GREATER_THAN_UV_MAX 0x02 +#endif + +#ifndef IS_NUMBER_NOT_INT +# define IS_NUMBER_NOT_INT 0x04 +#endif + +#ifndef IS_NUMBER_NEG +# define IS_NUMBER_NEG 0x08 +#endif + +#ifndef IS_NUMBER_INFINITY +# define IS_NUMBER_INFINITY 0x10 +#endif + +#ifndef IS_NUMBER_NAN +# define IS_NUMBER_NAN 0x20 +#endif +#ifndef GROK_NUMERIC_RADIX +# define GROK_NUMERIC_RADIX(sp, send) grok_numeric_radix(sp, send) +#endif +#ifndef PERL_SCAN_GREATER_THAN_UV_MAX +# define PERL_SCAN_GREATER_THAN_UV_MAX 0x02 +#endif + +#ifndef PERL_SCAN_SILENT_ILLDIGIT +# define PERL_SCAN_SILENT_ILLDIGIT 0x04 +#endif + +#ifndef PERL_SCAN_ALLOW_UNDERSCORES +# define PERL_SCAN_ALLOW_UNDERSCORES 0x01 +#endif + +#ifndef PERL_SCAN_DISALLOW_PREFIX +# define PERL_SCAN_DISALLOW_PREFIX 0x02 +#endif + +#ifndef grok_numeric_radix +#if defined(NEED_grok_numeric_radix) +static bool DPPP_(my_grok_numeric_radix)(pTHX_ const char ** sp, const char * send); +static +#else +extern bool DPPP_(my_grok_numeric_radix)(pTHX_ const char ** sp, const char * send); +#endif + +#ifdef grok_numeric_radix +# undef grok_numeric_radix +#endif +#define grok_numeric_radix(a,b) DPPP_(my_grok_numeric_radix)(aTHX_ a,b) +#define Perl_grok_numeric_radix DPPP_(my_grok_numeric_radix) + +#if defined(NEED_grok_numeric_radix) || defined(NEED_grok_numeric_radix_GLOBAL) +bool +DPPP_(my_grok_numeric_radix)(pTHX_ const char **sp, const char *send) +{ +#ifdef USE_LOCALE_NUMERIC +#ifdef PL_numeric_radix_sv + if (PL_numeric_radix_sv && IN_LOCALE) { + STRLEN len; + char* radix = SvPV(PL_numeric_radix_sv, len); + if (*sp + len <= send && memEQ(*sp, radix, len)) { + *sp += len; + return TRUE; + } + } +#else + /* older perls don't have PL_numeric_radix_sv so the radix + * must manually be requested from locale.h + */ +#include + dTHR; /* needed for older threaded perls */ + struct lconv *lc = localeconv(); + char *radix = lc->decimal_point; + if (radix && IN_LOCALE) { + STRLEN len = strlen(radix); + if (*sp + len <= send && memEQ(*sp, radix, len)) { + *sp += len; + return TRUE; + } + } +#endif +#endif /* USE_LOCALE_NUMERIC */ + /* always try "." if numeric radix didn't match because + * we may have data from different locales mixed */ + if (*sp < send && **sp == '.') { + ++*sp; + return TRUE; + } + return FALSE; +} +#endif +#endif + +#ifndef grok_number +#if defined(NEED_grok_number) +static int DPPP_(my_grok_number)(pTHX_ const char * pv, STRLEN len, UV * valuep); +static +#else +extern int DPPP_(my_grok_number)(pTHX_ const char * pv, STRLEN len, UV * valuep); +#endif + +#ifdef grok_number +# undef grok_number +#endif +#define grok_number(a,b,c) DPPP_(my_grok_number)(aTHX_ a,b,c) +#define Perl_grok_number DPPP_(my_grok_number) + +#if defined(NEED_grok_number) || defined(NEED_grok_number_GLOBAL) +int +DPPP_(my_grok_number)(pTHX_ const char *pv, STRLEN len, UV *valuep) +{ + const char *s = pv; + const char *send = pv + len; + const UV max_div_10 = UV_MAX / 10; + const char max_mod_10 = UV_MAX % 10; + int numtype = 0; + int sawinf = 0; + int sawnan = 0; + + while (s < send && isSPACE(*s)) + s++; + if (s == send) { + return 0; + } else if (*s == '-') { + s++; + numtype = IS_NUMBER_NEG; + } + else if (*s == '+') + s++; + + if (s == send) + return 0; + + /* next must be digit or the radix separator or beginning of infinity */ + if (isDIGIT(*s)) { + /* UVs are at least 32 bits, so the first 9 decimal digits cannot + overflow. */ + UV value = *s - '0'; + /* This construction seems to be more optimiser friendly. + (without it gcc does the isDIGIT test and the *s - '0' separately) + With it gcc on arm is managing 6 instructions (6 cycles) per digit. + In theory the optimiser could deduce how far to unroll the loop + before checking for overflow. */ + if (++s < send) { + int digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + /* Now got 9 digits, so need to check + each time for overflow. */ + digit = *s - '0'; + while (digit >= 0 && digit <= 9 + && (value < max_div_10 + || (value == max_div_10 + && digit <= max_mod_10))) { + value = value * 10 + digit; + if (++s < send) + digit = *s - '0'; + else + break; + } + if (digit >= 0 && digit <= 9 + && (s < send)) { + /* value overflowed. + skip the remaining digits, don't + worry about setting *valuep. */ + do { + s++; + } while (s < send && isDIGIT(*s)); + numtype |= + IS_NUMBER_GREATER_THAN_UV_MAX; + goto skip_value; + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + numtype |= IS_NUMBER_IN_UV; + if (valuep) + *valuep = value; + + skip_value: + if (GROK_NUMERIC_RADIX(&s, send)) { + numtype |= IS_NUMBER_NOT_INT; + while (s < send && isDIGIT(*s)) /* optional digits after the radix */ + s++; + } + } + else if (GROK_NUMERIC_RADIX(&s, send)) { + numtype |= IS_NUMBER_NOT_INT | IS_NUMBER_IN_UV; /* valuep assigned below */ + /* no digits before the radix means we need digits after it */ + if (s < send && isDIGIT(*s)) { + do { + s++; + } while (s < send && isDIGIT(*s)); + if (valuep) { + /* integer approximation is valid - it's 0. */ + *valuep = 0; + } + } + else + return 0; + } else if (*s == 'I' || *s == 'i') { + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; if (s == send || (*s != 'F' && *s != 'f')) return 0; + s++; if (s < send && (*s == 'I' || *s == 'i')) { + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; if (s == send || (*s != 'I' && *s != 'i')) return 0; + s++; if (s == send || (*s != 'T' && *s != 't')) return 0; + s++; if (s == send || (*s != 'Y' && *s != 'y')) return 0; + s++; + } + sawinf = 1; + } else if (*s == 'N' || *s == 'n') { + /* XXX TODO: There are signaling NaNs and quiet NaNs. */ + s++; if (s == send || (*s != 'A' && *s != 'a')) return 0; + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; + sawnan = 1; + } else + return 0; + + if (sawinf) { + numtype &= IS_NUMBER_NEG; /* Keep track of sign */ + numtype |= IS_NUMBER_INFINITY | IS_NUMBER_NOT_INT; + } else if (sawnan) { + numtype &= IS_NUMBER_NEG; /* Keep track of sign */ + numtype |= IS_NUMBER_NAN | IS_NUMBER_NOT_INT; + } else if (s < send) { + /* we can have an optional exponent part */ + if (*s == 'e' || *s == 'E') { + /* The only flag we keep is sign. Blow away any "it's UV" */ + numtype &= IS_NUMBER_NEG; + numtype |= IS_NUMBER_NOT_INT; + s++; + if (s < send && (*s == '-' || *s == '+')) + s++; + if (s < send && isDIGIT(*s)) { + do { + s++; + } while (s < send && isDIGIT(*s)); + } + else + return 0; + } + } + while (s < send && isSPACE(*s)) + s++; + if (s >= send) + return numtype; + if (len == 10 && memEQ(pv, "0 but true", 10)) { + if (valuep) + *valuep = 0; + return IS_NUMBER_IN_UV; + } + return 0; +} +#endif +#endif + +/* + * The grok_* routines have been modified to use warn() instead of + * Perl_warner(). Also, 'hexdigit' was the former name of PL_hexdigit, + * which is why the stack variable has been renamed to 'xdigit'. + */ + +#ifndef grok_bin +#if defined(NEED_grok_bin) +static UV DPPP_(my_grok_bin)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +static +#else +extern UV DPPP_(my_grok_bin)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +#endif + +#ifdef grok_bin +# undef grok_bin +#endif +#define grok_bin(a,b,c,d) DPPP_(my_grok_bin)(aTHX_ a,b,c,d) +#define Perl_grok_bin DPPP_(my_grok_bin) + +#if defined(NEED_grok_bin) || defined(NEED_grok_bin_GLOBAL) +UV +DPPP_(my_grok_bin)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) +{ + const char *s = start; + STRLEN len = *len_p; + UV value = 0; + NV value_nv = 0; + + const UV max_div_2 = UV_MAX / 2; + bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; + bool overflowed = FALSE; + + if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) { + /* strip off leading b or 0b. + for compatibility silently suffer "b" and "0b" as valid binary + numbers. */ + if (len >= 1) { + if (s[0] == 'b') { + s++; + len--; + } + else if (len >= 2 && s[0] == '0' && s[1] == 'b') { + s+=2; + len-=2; + } + } + } + + for (; len-- && *s; s++) { + char bit = *s; + if (bit == '0' || bit == '1') { + /* Write it in this wonky order with a goto to attempt to get the + compiler to make the common case integer-only loop pretty tight. + With gcc seems to be much straighter code than old scan_bin. */ + redo: + if (!overflowed) { + if (value <= max_div_2) { + value = (value << 1) | (bit - '0'); + continue; + } + /* Bah. We're just overflowed. */ + warn("Integer overflow in binary number"); + overflowed = TRUE; + value_nv = (NV) value; + } + value_nv *= 2.0; + /* If an NV has not enough bits in its mantissa to + * represent a UV this summing of small low-order numbers + * is a waste of time (because the NV cannot preserve + * the low-order bits anyway): we could just remember when + * did we overflow and in the end just multiply value_nv by the + * right amount. */ + value_nv += (NV)(bit - '0'); + continue; + } + if (bit == '_' && len && allow_underscores && (bit = s[1]) + && (bit == '0' || bit == '1')) + { + --len; + ++s; + goto redo; + } + if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) + warn("Illegal binary digit '%c' ignored", *s); + break; + } + + if ( ( overflowed && value_nv > 4294967295.0) +#if UVSIZE > 4 + || (!overflowed && value > 0xffffffff ) +#endif + ) { + warn("Binary number > 0b11111111111111111111111111111111 non-portable"); + } + *len_p = s - start; + if (!overflowed) { + *flags = 0; + return value; + } + *flags = PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) + *result = value_nv; + return UV_MAX; +} +#endif +#endif + +#ifndef grok_hex +#if defined(NEED_grok_hex) +static UV DPPP_(my_grok_hex)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +static +#else +extern UV DPPP_(my_grok_hex)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +#endif + +#ifdef grok_hex +# undef grok_hex +#endif +#define grok_hex(a,b,c,d) DPPP_(my_grok_hex)(aTHX_ a,b,c,d) +#define Perl_grok_hex DPPP_(my_grok_hex) + +#if defined(NEED_grok_hex) || defined(NEED_grok_hex_GLOBAL) +UV +DPPP_(my_grok_hex)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) +{ + const char *s = start; + STRLEN len = *len_p; + UV value = 0; + NV value_nv = 0; + + const UV max_div_16 = UV_MAX / 16; + bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; + bool overflowed = FALSE; + const char *xdigit; + + if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) { + /* strip off leading x or 0x. + for compatibility silently suffer "x" and "0x" as valid hex numbers. + */ + if (len >= 1) { + if (s[0] == 'x') { + s++; + len--; + } + else if (len >= 2 && s[0] == '0' && s[1] == 'x') { + s+=2; + len-=2; + } + } + } + + for (; len-- && *s; s++) { + xdigit = strchr((char *) PL_hexdigit, *s); + if (xdigit) { + /* Write it in this wonky order with a goto to attempt to get the + compiler to make the common case integer-only loop pretty tight. + With gcc seems to be much straighter code than old scan_hex. */ + redo: + if (!overflowed) { + if (value <= max_div_16) { + value = (value << 4) | ((xdigit - PL_hexdigit) & 15); + continue; + } + warn("Integer overflow in hexadecimal number"); + overflowed = TRUE; + value_nv = (NV) value; + } + value_nv *= 16.0; + /* If an NV has not enough bits in its mantissa to + * represent a UV this summing of small low-order numbers + * is a waste of time (because the NV cannot preserve + * the low-order bits anyway): we could just remember when + * did we overflow and in the end just multiply value_nv by the + * right amount of 16-tuples. */ + value_nv += (NV)((xdigit - PL_hexdigit) & 15); + continue; + } + if (*s == '_' && len && allow_underscores && s[1] + && (xdigit = strchr((char *) PL_hexdigit, s[1]))) + { + --len; + ++s; + goto redo; + } + if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) + warn("Illegal hexadecimal digit '%c' ignored", *s); + break; + } + + if ( ( overflowed && value_nv > 4294967295.0) +#if UVSIZE > 4 + || (!overflowed && value > 0xffffffff ) +#endif + ) { + warn("Hexadecimal number > 0xffffffff non-portable"); + } + *len_p = s - start; + if (!overflowed) { + *flags = 0; + return value; + } + *flags = PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) + *result = value_nv; + return UV_MAX; +} +#endif +#endif + +#ifndef grok_oct +#if defined(NEED_grok_oct) +static UV DPPP_(my_grok_oct)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +static +#else +extern UV DPPP_(my_grok_oct)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +#endif + +#ifdef grok_oct +# undef grok_oct +#endif +#define grok_oct(a,b,c,d) DPPP_(my_grok_oct)(aTHX_ a,b,c,d) +#define Perl_grok_oct DPPP_(my_grok_oct) + +#if defined(NEED_grok_oct) || defined(NEED_grok_oct_GLOBAL) +UV +DPPP_(my_grok_oct)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) +{ + const char *s = start; + STRLEN len = *len_p; + UV value = 0; + NV value_nv = 0; + + const UV max_div_8 = UV_MAX / 8; + bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; + bool overflowed = FALSE; + + for (; len-- && *s; s++) { + /* gcc 2.95 optimiser not smart enough to figure that this subtraction + out front allows slicker code. */ + int digit = *s - '0'; + if (digit >= 0 && digit <= 7) { + /* Write it in this wonky order with a goto to attempt to get the + compiler to make the common case integer-only loop pretty tight. + */ + redo: + if (!overflowed) { + if (value <= max_div_8) { + value = (value << 3) | digit; + continue; + } + /* Bah. We're just overflowed. */ + warn("Integer overflow in octal number"); + overflowed = TRUE; + value_nv = (NV) value; + } + value_nv *= 8.0; + /* If an NV has not enough bits in its mantissa to + * represent a UV this summing of small low-order numbers + * is a waste of time (because the NV cannot preserve + * the low-order bits anyway): we could just remember when + * did we overflow and in the end just multiply value_nv by the + * right amount of 8-tuples. */ + value_nv += (NV)digit; + continue; + } + if (digit == ('_' - '0') && len && allow_underscores + && (digit = s[1] - '0') && (digit >= 0 && digit <= 7)) + { + --len; + ++s; + goto redo; + } + /* Allow \octal to work the DWIM way (that is, stop scanning + * as soon as non-octal characters are seen, complain only iff + * someone seems to want to use the digits eight and nine). */ + if (digit == 8 || digit == 9) { + if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) + warn("Illegal octal digit '%c' ignored", *s); + } + break; + } + + if ( ( overflowed && value_nv > 4294967295.0) +#if UVSIZE > 4 + || (!overflowed && value > 0xffffffff ) +#endif + ) { + warn("Octal number > 037777777777 non-portable"); + } + *len_p = s - start; + if (!overflowed) { + *flags = 0; + return value; + } + *flags = PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) + *result = value_nv; + return UV_MAX; +} +#endif +#endif + +#if !defined(my_snprintf) +#if defined(NEED_my_snprintf) +static int DPPP_(my_my_snprintf)(char * buffer, const Size_t len, const char * format, ...); +static +#else +extern int DPPP_(my_my_snprintf)(char * buffer, const Size_t len, const char * format, ...); +#endif + +#define my_snprintf DPPP_(my_my_snprintf) +#define Perl_my_snprintf DPPP_(my_my_snprintf) + +#if defined(NEED_my_snprintf) || defined(NEED_my_snprintf_GLOBAL) + +int +DPPP_(my_my_snprintf)(char *buffer, const Size_t len, const char *format, ...) +{ + dTHX; + int retval; + va_list ap; + va_start(ap, format); +#ifdef HAS_VSNPRINTF + retval = vsnprintf(buffer, len, format, ap); +#else + retval = vsprintf(buffer, format, ap); +#endif + va_end(ap); + if (retval < 0 || (len > 0 && (Size_t)retval >= len)) + Perl_croak(aTHX_ "panic: my_snprintf buffer overflow"); + return retval; +} + +#endif +#endif + +#if !defined(my_sprintf) +#if defined(NEED_my_sprintf) +static int DPPP_(my_my_sprintf)(char * buffer, const char * pat, ...); +static +#else +extern int DPPP_(my_my_sprintf)(char * buffer, const char * pat, ...); +#endif + +#define my_sprintf DPPP_(my_my_sprintf) +#define Perl_my_sprintf DPPP_(my_my_sprintf) + +#if defined(NEED_my_sprintf) || defined(NEED_my_sprintf_GLOBAL) + +int +DPPP_(my_my_sprintf)(char *buffer, const char* pat, ...) +{ + va_list args; + va_start(args, pat); + vsprintf(buffer, pat, args); + va_end(args); + return strlen(buffer); +} + +#endif +#endif + +#ifdef NO_XSLOCKS +# ifdef dJMPENV +# define dXCPT dJMPENV; int rEtV = 0 +# define XCPT_TRY_START JMPENV_PUSH(rEtV); if (rEtV == 0) +# define XCPT_TRY_END JMPENV_POP; +# define XCPT_CATCH if (rEtV != 0) +# define XCPT_RETHROW JMPENV_JUMP(rEtV) +# else +# define dXCPT Sigjmp_buf oldTOP; int rEtV = 0 +# define XCPT_TRY_START Copy(top_env, oldTOP, 1, Sigjmp_buf); rEtV = Sigsetjmp(top_env, 1); if (rEtV == 0) +# define XCPT_TRY_END Copy(oldTOP, top_env, 1, Sigjmp_buf); +# define XCPT_CATCH if (rEtV != 0) +# define XCPT_RETHROW Siglongjmp(top_env, rEtV) +# endif +#endif + +#if !defined(my_strlcat) +#if defined(NEED_my_strlcat) +static Size_t DPPP_(my_my_strlcat)(char * dst, const char * src, Size_t size); +static +#else +extern Size_t DPPP_(my_my_strlcat)(char * dst, const char * src, Size_t size); +#endif + +#define my_strlcat DPPP_(my_my_strlcat) +#define Perl_my_strlcat DPPP_(my_my_strlcat) + +#if defined(NEED_my_strlcat) || defined(NEED_my_strlcat_GLOBAL) + +Size_t +DPPP_(my_my_strlcat)(char *dst, const char *src, Size_t size) +{ + Size_t used, length, copy; + + used = strlen(dst); + length = strlen(src); + if (size > 0 && used < size - 1) { + copy = (length >= size - used) ? size - used - 1 : length; + memcpy(dst + used, src, copy); + dst[used + copy] = '\0'; + } + return used + length; +} +#endif +#endif + +#if !defined(my_strlcpy) +#if defined(NEED_my_strlcpy) +static Size_t DPPP_(my_my_strlcpy)(char * dst, const char * src, Size_t size); +static +#else +extern Size_t DPPP_(my_my_strlcpy)(char * dst, const char * src, Size_t size); +#endif + +#define my_strlcpy DPPP_(my_my_strlcpy) +#define Perl_my_strlcpy DPPP_(my_my_strlcpy) + +#if defined(NEED_my_strlcpy) || defined(NEED_my_strlcpy_GLOBAL) + +Size_t +DPPP_(my_my_strlcpy)(char *dst, const char *src, Size_t size) +{ + Size_t length, copy; + + length = strlen(src); + if (size > 0) { + copy = (length >= size) ? size - 1 : length; + memcpy(dst, src, copy); + dst[copy] = '\0'; + } + return length; +} + +#endif +#endif +#ifndef PERL_PV_ESCAPE_QUOTE +# define PERL_PV_ESCAPE_QUOTE 0x0001 +#endif + +#ifndef PERL_PV_PRETTY_QUOTE +# define PERL_PV_PRETTY_QUOTE PERL_PV_ESCAPE_QUOTE +#endif + +#ifndef PERL_PV_PRETTY_ELLIPSES +# define PERL_PV_PRETTY_ELLIPSES 0x0002 +#endif + +#ifndef PERL_PV_PRETTY_LTGT +# define PERL_PV_PRETTY_LTGT 0x0004 +#endif + +#ifndef PERL_PV_ESCAPE_FIRSTCHAR +# define PERL_PV_ESCAPE_FIRSTCHAR 0x0008 +#endif + +#ifndef PERL_PV_ESCAPE_UNI +# define PERL_PV_ESCAPE_UNI 0x0100 +#endif + +#ifndef PERL_PV_ESCAPE_UNI_DETECT +# define PERL_PV_ESCAPE_UNI_DETECT 0x0200 +#endif + +#ifndef PERL_PV_ESCAPE_ALL +# define PERL_PV_ESCAPE_ALL 0x1000 +#endif + +#ifndef PERL_PV_ESCAPE_NOBACKSLASH +# define PERL_PV_ESCAPE_NOBACKSLASH 0x2000 +#endif + +#ifndef PERL_PV_ESCAPE_NOCLEAR +# define PERL_PV_ESCAPE_NOCLEAR 0x4000 +#endif + +#ifndef PERL_PV_ESCAPE_RE +# define PERL_PV_ESCAPE_RE 0x8000 +#endif + +#ifndef PERL_PV_PRETTY_NOCLEAR +# define PERL_PV_PRETTY_NOCLEAR PERL_PV_ESCAPE_NOCLEAR +#endif +#ifndef PERL_PV_PRETTY_DUMP +# define PERL_PV_PRETTY_DUMP PERL_PV_PRETTY_ELLIPSES|PERL_PV_PRETTY_QUOTE +#endif + +#ifndef PERL_PV_PRETTY_REGPROP +# define PERL_PV_PRETTY_REGPROP PERL_PV_PRETTY_ELLIPSES|PERL_PV_PRETTY_LTGT|PERL_PV_ESCAPE_RE +#endif + +/* Hint: pv_escape + * Note that unicode functionality is only backported to + * those perl versions that support it. For older perl + * versions, the implementation will fall back to bytes. + */ + +#ifndef pv_escape +#if defined(NEED_pv_escape) +static char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags); +static +#else +extern char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags); +#endif + +#ifdef pv_escape +# undef pv_escape +#endif +#define pv_escape(a,b,c,d,e,f) DPPP_(my_pv_escape)(aTHX_ a,b,c,d,e,f) +#define Perl_pv_escape DPPP_(my_pv_escape) + +#if defined(NEED_pv_escape) || defined(NEED_pv_escape_GLOBAL) + +char * +DPPP_(my_pv_escape)(pTHX_ SV *dsv, char const * const str, + const STRLEN count, const STRLEN max, + STRLEN * const escaped, const U32 flags) +{ + const char esc = flags & PERL_PV_ESCAPE_RE ? '%' : '\\'; + const char dq = flags & PERL_PV_ESCAPE_QUOTE ? '"' : esc; + char octbuf[32] = "%123456789ABCDF"; + STRLEN wrote = 0; + STRLEN chsize = 0; + STRLEN readsize = 1; +#if defined(is_utf8_string) && defined(utf8_to_uvchr) + bool isuni = flags & PERL_PV_ESCAPE_UNI ? 1 : 0; +#endif + const char *pv = str; + const char * const end = pv + count; + octbuf[0] = esc; + + if (!(flags & PERL_PV_ESCAPE_NOCLEAR)) + sv_setpvs(dsv, ""); + +#if defined(is_utf8_string) && defined(utf8_to_uvchr) + if ((flags & PERL_PV_ESCAPE_UNI_DETECT) && is_utf8_string((U8*)pv, count)) + isuni = 1; +#endif + + for (; pv < end && (!max || wrote < max) ; pv += readsize) { + const UV u = +#if defined(is_utf8_string) && defined(utf8_to_uvchr) + isuni ? utf8_to_uvchr((U8*)pv, &readsize) : +#endif + (U8)*pv; + const U8 c = (U8)u & 0xFF; + + if (u > 255 || (flags & PERL_PV_ESCAPE_ALL)) { + if (flags & PERL_PV_ESCAPE_FIRSTCHAR) + chsize = my_snprintf(octbuf, sizeof octbuf, + "%"UVxf, u); + else + chsize = my_snprintf(octbuf, sizeof octbuf, + "%cx{%"UVxf"}", esc, u); + } else if (flags & PERL_PV_ESCAPE_NOBACKSLASH) { + chsize = 1; + } else { + if (c == dq || c == esc || !isPRINT(c)) { + chsize = 2; + switch (c) { + case '\\' : /* fallthrough */ + case '%' : if (c == esc) + octbuf[1] = esc; + else + chsize = 1; + break; + case '\v' : octbuf[1] = 'v'; break; + case '\t' : octbuf[1] = 't'; break; + case '\r' : octbuf[1] = 'r'; break; + case '\n' : octbuf[1] = 'n'; break; + case '\f' : octbuf[1] = 'f'; break; + case '"' : if (dq == '"') + octbuf[1] = '"'; + else + chsize = 1; + break; + default: chsize = my_snprintf(octbuf, sizeof octbuf, + pv < end && isDIGIT((U8)*(pv+readsize)) + ? "%c%03o" : "%c%o", esc, c); + } + } else { + chsize = 1; + } + } + if (max && wrote + chsize > max) { + break; + } else if (chsize > 1) { + sv_catpvn(dsv, octbuf, chsize); + wrote += chsize; + } else { + char tmp[2]; + my_snprintf(tmp, sizeof tmp, "%c", c); + sv_catpvn(dsv, tmp, 1); + wrote++; + } + if (flags & PERL_PV_ESCAPE_FIRSTCHAR) + break; + } + if (escaped != NULL) + *escaped= pv - str; + return SvPVX(dsv); +} + +#endif +#endif + +#ifndef pv_pretty +#if defined(NEED_pv_pretty) +static char * DPPP_(my_pv_pretty)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, char const * const start_color, char const * const end_color, const U32 flags); +static +#else +extern char * DPPP_(my_pv_pretty)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, char const * const start_color, char const * const end_color, const U32 flags); +#endif + +#ifdef pv_pretty +# undef pv_pretty +#endif +#define pv_pretty(a,b,c,d,e,f,g) DPPP_(my_pv_pretty)(aTHX_ a,b,c,d,e,f,g) +#define Perl_pv_pretty DPPP_(my_pv_pretty) + +#if defined(NEED_pv_pretty) || defined(NEED_pv_pretty_GLOBAL) + +char * +DPPP_(my_pv_pretty)(pTHX_ SV *dsv, char const * const str, const STRLEN count, + const STRLEN max, char const * const start_color, char const * const end_color, + const U32 flags) +{ + const U8 dq = (flags & PERL_PV_PRETTY_QUOTE) ? '"' : '%'; + STRLEN escaped; + + if (!(flags & PERL_PV_PRETTY_NOCLEAR)) + sv_setpvs(dsv, ""); + + if (dq == '"') + sv_catpvs(dsv, "\""); + else if (flags & PERL_PV_PRETTY_LTGT) + sv_catpvs(dsv, "<"); + + if (start_color != NULL) + sv_catpv(dsv, D_PPP_CONSTPV_ARG(start_color)); + + pv_escape(dsv, str, count, max, &escaped, flags | PERL_PV_ESCAPE_NOCLEAR); + + if (end_color != NULL) + sv_catpv(dsv, D_PPP_CONSTPV_ARG(end_color)); + + if (dq == '"') + sv_catpvs(dsv, "\""); + else if (flags & PERL_PV_PRETTY_LTGT) + sv_catpvs(dsv, ">"); + + if ((flags & PERL_PV_PRETTY_ELLIPSES) && escaped < count) + sv_catpvs(dsv, "..."); + + return SvPVX(dsv); +} + +#endif +#endif + +#ifndef pv_display +#if defined(NEED_pv_display) +static char * DPPP_(my_pv_display)(pTHX_ SV * dsv, const char * pv, STRLEN cur, STRLEN len, STRLEN pvlim); +static +#else +extern char * DPPP_(my_pv_display)(pTHX_ SV * dsv, const char * pv, STRLEN cur, STRLEN len, STRLEN pvlim); +#endif + +#ifdef pv_display +# undef pv_display +#endif +#define pv_display(a,b,c,d,e) DPPP_(my_pv_display)(aTHX_ a,b,c,d,e) +#define Perl_pv_display DPPP_(my_pv_display) + +#if defined(NEED_pv_display) || defined(NEED_pv_display_GLOBAL) + +char * +DPPP_(my_pv_display)(pTHX_ SV *dsv, const char *pv, STRLEN cur, STRLEN len, STRLEN pvlim) +{ + pv_pretty(dsv, pv, cur, pvlim, NULL, NULL, PERL_PV_PRETTY_DUMP); + if (len > cur && pv[cur] == '\0') + sv_catpvs(dsv, "\\0"); + return SvPVX(dsv); +} + +#endif +#endif + +#endif /* _P_P_PORTABILITY_H_ */ + +/* End of File ppport.h */ diff --git a/perl/t/00_compile.t b/perl/t/00_compile.t new file mode 100644 index 0000000..66fe8f0 --- /dev/null +++ b/perl/t/00_compile.t @@ -0,0 +1,6 @@ +use strict; +use warnings; +use Test::More tests => 1; + +use_ok 'Data::MessagePack'; + diff --git a/perl/t/01_pack.t b/perl/t/01_pack.t new file mode 100644 index 0000000..0917f04 --- /dev/null +++ b/perl/t/01_pack.t @@ -0,0 +1,61 @@ +use t::Util; +use Test::More; +use Data::MessagePack; + +sub packit { + local $_ = unpack("H*", Data::MessagePack->pack($_[0])); + s/(..)/$1 /g; + s/ $//; + $_; +} + +sub pis ($$) { + is packit($_[0]), $_[1], 'dump ' . $_[1]; +} + +my @dat = ( + 0, '00', + 1, '01', + 127, '7f', + 128, 'cc 80', + 255, 'cc ff', + 256, 'cd 01 00', + 65535, 'cd ff ff', + 65536, 'ce 00 01 00 00', + -1, 'ff', + -32, 'e0', + -33, 'd0 df', + -128, 'd0 80', + -129, 'd1 ff 7f', + -32768, 'd1 80 00', + -32769, 'd2 ff ff 7f ff', + 1.0, 'cb 3f f0 00 00 00 00 00 00', + "", 'a0', + "a", 'a1 61', + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 'bf 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61', + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 'da 00 20 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61', + undef, 'c0', + Data::MessagePack::true(), 'c3', + Data::MessagePack::false(), 'c2', + [], '90', + [+[]], '91 90', + [[], undef], '92 90 c0', + {'a', 0}, '81 a1 61 00', + 8388608, 'ce 00 80 00 00', + + [undef, false, true], '93 c0 c2 c3', + ["", "a", "bc", "def"], '94 a0 a1 61 a2 62 63 a3 64 65 66', + [[], [[undef]]], '92 90 91 91 c0', + [undef, false, true], '93 c0 c2 c3', + [[0, 64, 127], [-32, -16, -1]], '92 93 00 40 7f 93 e0 f0 ff', + [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], '99 00 d0 80 ff 00 d1 80 00 ff 00 d2 80 00 00 00 ff', + 2147483648, 'ce 80 00 00 00', + -2147483648, 'd2 80 00 00 00', +); +@dat = (2147483648, 'ce 80 00 00 00'); +plan tests => 1*(scalar(@dat)/2); + +for (my $i=0; $iunpack($v); +} + +sub pis ($$) { + is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0]; +} + +my @dat = do 't/data.pl'; + +plan tests => 1*(scalar(@dat)/2); + +for (my $i=0; $inew; +sub unpackit { + my $v = $_[0]; + $v =~ s/ //g; + $v = pack 'H*', $v; + $up->reset; + my $ret = $up->execute($v, 0); + if ($ret != length($v)) { + fail "extra bytes"; + } + return $up->data; +} + +sub pis ($$) { + is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0]; +} + +my @dat = do 't/data.pl'; + +plan tests => 1*(scalar(@dat)/2) + 1; + +isa_ok $up, 'Data::MessagePack::Unpacker'; +for (my $i=0; $iunpack( + Data::MessagePack->pack($_[0]), + ); +} + +sub pis ($) { + is_deeply invert($_[0]), $_[0], 'dump ' . $_[0]; +} + +my @dat = do 't/data.pl'; + +plan tests => 1*(scalar(@dat)/2); + +for (my $i=0; $iimport; + warnings->import; + + no strict 'refs'; + *{"$pkg\::true"} = sub () { + Data::MessagePack::true() + }; + *{"$pkg\::false"} = sub () { + Data::MessagePack::false() + }; +} + +1; diff --git a/perl/t/data.pl b/perl/t/data.pl new file mode 100644 index 0000000..1a2b2b4 --- /dev/null +++ b/perl/t/data.pl @@ -0,0 +1,15 @@ +no warnings 'uninitialized'; # i need this. i need this. +( + '93 c0 c2 c3' => [undef, false, true], + '94 a0 a1 61 a2 62 63 a3 64 65 66', ["", "a", "bc", "def"], + '92 90 91 91 c0', [[], [[undef]]], + '93 c0 c2 c3', [undef, false, true], + 'ce 80 00 00 00', 2147483648, + '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff', [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + '92 93 00 40 7f 93 e0 f0 ff', [[0, 64, 127], [-32, -16, -1]], + '96 dc 00 00 dc 00 01 c0 dc 00 02 c2 c3 dd 00 00 00 00 dd 00 00 00 01 c0 dd 00 00 00 02 c2 c3', [[], [undef], [false, true], [], [undef], [false, true]], + '96 da 00 00 da 00 01 61 da 00 02 61 62 db 00 00 00 00 db 00 00 00 01 61 db 00 00 00 02 61 62', ["", "a", "ab", "", "a", "ab"], + '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff', [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], + '82 c2 81 c0 c0 c3 81 c0 80', {false,{undef,undef}, true,{undef,{}}}, + '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2', [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}], +) diff --git a/perl/unpack.c b/perl/unpack.c new file mode 100644 index 0000000..e5e069e --- /dev/null +++ b/perl/unpack.c @@ -0,0 +1,268 @@ +#ifdef __cplusplus +extern "C" { +#endif + +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "ppport.h" + +#ifdef __cplusplus +}; +#endif + +typedef struct { + int finished; + SV* source; +} unpack_user; + +#include "msgpack/unpack_define.h" + +#define msgpack_unpack_struct(name) \ + struct template ## name + +#define msgpack_unpack_func(ret, name) \ + ret template ## name + +#define msgpack_unpack_callback(name) \ + template_callback ## name + +#define msgpack_unpack_object SV* + +#define msgpack_unpack_user unpack_user + +struct template_context; +typedef struct template_context msgpack_unpack_t; + +static void template_init(msgpack_unpack_t* u); + +static SV* template_data(msgpack_unpack_t* u); + +static int template_execute(msgpack_unpack_t* u, + const char* data, size_t len, size_t* off); + +static inline SV* template_callback_root(unpack_user* u) +{ return &PL_sv_undef; } + +static inline int template_callback_uint8(unpack_user* u, uint8_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_uint16(unpack_user* u, uint16_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_uint32(unpack_user* u, uint32_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_int8(unpack_user* u, int8_t d, SV** o) +{ *o = newSViv((long)d); return 0; } + +static inline int template_callback_int16(unpack_user* u, int16_t d, SV** o) +{ *o = newSViv((long)d); return 0; } + +static inline int template_callback_int32(unpack_user* u, int32_t d, SV** o) +{ *o = newSViv((long)d); return 0; } + +static inline int template_callback_int64(unpack_user* u, int64_t d, SV** o) +{ *o = newSViv(d); return 0; } + +static inline int template_callback_float(unpack_user* u, float d, SV** o) +{ *o = newSVnv(d); return 0; } + +static inline int template_callback_double(unpack_user* u, double d, SV** o) +{ *o = newSVnv(d); return 0; } + +static inline int template_callback_nil(unpack_user* u, SV** o) +{ *o = &PL_sv_undef; return 0; } + +static inline int template_callback_true(unpack_user* u, SV** o) +{ *o = &PL_sv_yes; return 0; } + +static inline int template_callback_false(unpack_user* u, SV** o) +{ *o = &PL_sv_no; return 0;} + +static inline int template_callback_array(unpack_user* u, unsigned int n, SV** o) +{ AV* a = newAV(); *o = (SV*)newRV_noinc((SV*)a); av_extend(a, n); return 0; } + +static inline int template_callback_array_item(unpack_user* u, SV** c, SV* o) +{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] + +static inline int template_callback_map(unpack_user* u, unsigned int n, SV** o) +{ HV * h = newHV(); *o = newRV_noinc((SV*)h); return 0; } + +static inline int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v) +{ hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; } + +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o) +{ *o = (l == 0) ? newSVpv("", 0) : newSVpv(p, l); return 0; } + +#define UNPACKER(from, name) \ + msgpack_unpack_t *name; \ + name = INT2PTR(msgpack_unpack_t*, SvROK((from)) ? SvIV(SvRV((from))) : SvIV((from))); \ + if(name == NULL) { \ + Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be."); \ + } + +#include "msgpack/unpack_template.h" + +SV* _msgpack_unpack(SV* data, int limit) { + msgpack_unpack_t mp; + unpack_user u = {0, &PL_sv_undef}; + int ret; + size_t from = 0; + STRLEN dlen; + const char * dptr = SvPV_const(data, dlen); + + template_init(&mp); + mp.user = u; + + mp.user.source = data; + ret = template_execute(&mp, dptr, (size_t)dlen, &from); + mp.user.source = &PL_sv_undef; + + if(ret < 0) { + Perl_croak(aTHX_ "parse error."); + } else if(ret == 0) { + Perl_croak(aTHX_ "insufficient bytes."); + } else { + if(from < dlen) { + Perl_croak(aTHX_ "extra bytes."); + } + return template_data(&mp); + } +} + +XS(xs_unpack_limit) { + dXSARGS; + + if (items != 3) { + Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata', $limit)"); + } + + { + int limit = SvIV(ST(2)); + ST(0) = _msgpack_unpack(ST(1), limit); + } + XSRETURN(1); +} + + +XS(xs_unpack) { + dXSARGS; + msgpack_unpack_t mp; + + if (items != 2) { + Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata')"); + } + + { + ST(0) = _msgpack_unpack(ST(1), sv_len(ST(1))); + } + + XSRETURN(1); +} + +/* ------------------------------ stream -- */ + +static void _reset(SV* self) { + UNPACKER(self, mp); + template_init(mp); + unpack_user u = {0, &PL_sv_undef}; + mp->user = u; +} + +XS(xs_unpacker_new) { + dXSARGS; + SV* self = sv_newmortal(); + msgpack_unpack_t *mp; + + Newx(mp, 1, msgpack_unpack_t); + + sv_setref_pv(self, "Data::MessagePack::Unpacker", mp); + _reset(self); + + ST(0) = self; + XSRETURN(1); +} + +static SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) { + UNPACKER(self, mp); + + size_t from = off; + const char* dptr = SvPV_nolen_const(data); + long dlen = limit; + int ret; + + if(from >= dlen) { + Perl_croak(aTHX_ "offset is bigger than data buffer size."); + } + + mp->user.source = data; + ret = template_execute(mp, dptr, (size_t)dlen, &from); + mp->user.source = &PL_sv_undef; + + if(ret < 0) { + Perl_croak(aTHX_ "parse error."); + } else if(ret > 0) { + mp->user.finished = 1; + return newSVuv(from); + } else { + mp->user.finished = 0; + return newSVuv(from); + } +} + +XS(xs_unpacker_execute) { + dXSARGS; + SV* self = ST(0); + SV* data = ST(1); + IV off = SvIV(ST(2)); + + ST(0) = _execute_impl(self, data, off, sv_len(data)); + + XSRETURN(1); +} + +XS(xs_unpacker_execute_limit) { + dXSARGS; + SV* self = ST(0); + SV* data = ST(1); + IV off = SvIV(ST(2)); + IV limit = SvIV(ST(3)); + + ST(0) = _execute_impl(self, data, off, limit); + + XSRETURN(1); +} + +XS(xs_unpacker_is_finished) { + dXSARGS; + + UNPACKER(ST(0), mp); + ST(0) = (mp->user.finished) ? &PL_sv_yes : &PL_sv_no; + + XSRETURN(1); +} + +XS(xs_unpacker_data) { + dXSARGS; + + UNPACKER(ST(0), mp); + ST(0) = template_data(mp); + + XSRETURN(1); +} + +XS(xs_unpacker_reset) { + dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: $unpacker->reset()"); + } + + _reset(ST(0)); + + XSRETURN(0); +} + diff --git a/perl/xt/99_pod.t b/perl/xt/99_pod.t new file mode 100644 index 0000000..437887a --- /dev/null +++ b/perl/xt/99_pod.t @@ -0,0 +1,4 @@ +use Test::More; +eval "use Test::Pod 1.00"; +plan skip_all => "Test::Pod 1.00 required for testing POD" if $@; +all_pod_files_ok(); From d449b1d20d680242553cd692a8d3e00112f6ae7e Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 12:55:41 +0900 Subject: [PATCH 0112/1172] added perl support --- perl/Makefile.PL | 34 + perl/MessagePack.c | 39 + perl/benchmark/p1.pl | 16 + perl/lib/Data/MessagePack.pm | 33 + perl/lib/Data/MessagePack/Unpacker.pod | 52 + perl/pack.c | 122 + perl/ppport.h | 6984 ++++++++++++++++++++++++ perl/t/00_compile.t | 6 + perl/t/01_pack.t | 61 + perl/t/02_unpack.t | 24 + perl/t/03_stream_unpack.t | 32 + perl/t/04_invert.t | 24 + perl/t/Util.pm | 20 + perl/t/data.pl | 15 + perl/unpack.c | 268 + perl/xt/99_pod.t | 4 + 16 files changed, 7734 insertions(+) create mode 100644 perl/Makefile.PL create mode 100644 perl/MessagePack.c create mode 100644 perl/benchmark/p1.pl create mode 100644 perl/lib/Data/MessagePack.pm create mode 100644 perl/lib/Data/MessagePack/Unpacker.pod create mode 100644 perl/pack.c create mode 100644 perl/ppport.h create mode 100644 perl/t/00_compile.t create mode 100644 perl/t/01_pack.t create mode 100644 perl/t/02_unpack.t create mode 100644 perl/t/03_stream_unpack.t create mode 100644 perl/t/04_invert.t create mode 100644 perl/t/Util.pm create mode 100644 perl/t/data.pl create mode 100644 perl/unpack.c create mode 100644 perl/xt/99_pod.t diff --git a/perl/Makefile.PL b/perl/Makefile.PL new file mode 100644 index 0000000..f60e125 --- /dev/null +++ b/perl/Makefile.PL @@ -0,0 +1,34 @@ +use inc::Module::Install; +name 'Data-MessagePack'; +all_from 'lib/Data/MessagePack.pm'; + +perl_version '5.008005'; +license 'perl'; +can_cc or die "This module requires a C compiler"; + +my $ccflags = '-I../ '; + +makemaker_args( + OBJECT => '$(O_FILES)', + LIBS => [''], + CCFLAGS => $ccflags, + clean => { + FILES => q{ + *.stackdump + *.gcov *.gcda *.gcno + *.out + nytprof + cover_db + }, + }, +); + +tests 't/*.t'; +author_tests('xt'); + +auto_set_repository; +build_requires 'Test::More'; +use_test_base; +auto_include; +WriteAll; + diff --git a/perl/MessagePack.c b/perl/MessagePack.c new file mode 100644 index 0000000..c40e46a --- /dev/null +++ b/perl/MessagePack.c @@ -0,0 +1,39 @@ +#ifdef __cplusplus +extern "C" { +#endif +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "ppport.h" +#ifdef __cplusplus +}; +#endif + +XS(xs_pack); +XS(xs_unpack); +XS(xs_unpacker_new); +XS(xs_unpacker_execute); +XS(xs_unpacker_execute_limit); +XS(xs_unpacker_is_finished); +XS(xs_unpacker_data); +XS(xs_unpacker_reset); + +XS(boot_Data__MessagePack) { + dXSARGS; + HV * stash; + + newXS("Data::MessagePack::pack", xs_pack, __FILE__); + newXS("Data::MessagePack::unpack", xs_unpack, __FILE__); + stash = gv_stashpvn("Data::MessagePack", strlen("Data::MessagePack"), TRUE); + newCONSTSUB(stash, "true", &PL_sv_yes); + newCONSTSUB(stash, "false", &PL_sv_no); + + newXS("Data::MessagePack::Unpacker::new", xs_unpacker_new, __FILE__); + newXS("Data::MessagePack::Unpacker::execute", xs_unpacker_execute, __FILE__); + newXS("Data::MessagePack::Unpacker::execute_limit", xs_unpacker_execute_limit, __FILE__); + newXS("Data::MessagePack::Unpacker::is_finished", xs_unpacker_is_finished, __FILE__); + newXS("Data::MessagePack::Unpacker::data", xs_unpacker_data, __FILE__); + newXS("Data::MessagePack::Unpacker::reset", xs_unpacker_reset, __FILE__); + +} + diff --git a/perl/benchmark/p1.pl b/perl/benchmark/p1.pl new file mode 100644 index 0000000..257932e --- /dev/null +++ b/perl/benchmark/p1.pl @@ -0,0 +1,16 @@ +use strict; +use warnings; +use Data::MessagePack; +use JSON::XS; +use Benchmark ':all'; + +my $a = [0..2**24]; + +print "-- serialize\n"; +cmpthese( + -1 => { + json => sub { JSON::XS::encode_json($a) }, + mp => sub { Data::MessagePack->pack($a) }, + } +); + diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm new file mode 100644 index 0000000..d26bf9e --- /dev/null +++ b/perl/lib/Data/MessagePack.pm @@ -0,0 +1,33 @@ +package Data::MessagePack; +use strict; +use warnings; +use XSLoader; + +our $VERSION = 0.01; + +XSLoader::load(__PACKAGE__, $VERSION); + +1; +__END__ + +=head1 NAME + +Data::MessagePack - messagepack + +=head1 SYNOPSIS + + my $packed = Data::MessagePack->pack($dat); + my $unpacked = Data::MessagePack->unpack($dat); + +=head1 DESCRIPTION + +Data::MessagePack is a binary packer for perl. + +=head1 AUTHORS + +Tokuhiro Matsuno + +=head1 SEE ALSO + +L + diff --git a/perl/lib/Data/MessagePack/Unpacker.pod b/perl/lib/Data/MessagePack/Unpacker.pod new file mode 100644 index 0000000..61cbd21 --- /dev/null +++ b/perl/lib/Data/MessagePack/Unpacker.pod @@ -0,0 +1,52 @@ +=head1 NAME + +Data::MessagePack::Unpacker - messagepack streaming deserializer + +=head1 SYNOPSIS + + use Data::Dumper; + my $up = Data::MessagePack::Unpacker->new; + my $ret = $up->execute($v, 0); + if ($ret != length($v)) { + fail "extra bytes"; + } + return Dumper($up->data); + +=head1 DESCRIPTION + +This is an streaming deserializer for messagepack. + +=head1 METHODS + +=over 4 + +=item my $up = Data::MessagePack::Unpacker->new() + +create new stream deserializer + +=item $up->execute() + +=item $up->execute_limit() + +=item $up->is_finished() + +is this deserializer finished? + +=item $up->data() + +returns deserialized object. + +=item $up->reset() + +reset the stream deserializer, without memory zone. + +=back + +=head1 AUTHORS + +Tokuhiro Matsuno + +=head1 SEE ALSO + +L + diff --git a/perl/pack.c b/perl/pack.c new file mode 100644 index 0000000..5bb667b --- /dev/null +++ b/perl/pack.c @@ -0,0 +1,122 @@ +#ifdef __cplusplus +extern "C" { +#endif +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "ppport.h" +#ifdef __cplusplus +}; +#endif + +#include "msgpack/pack_define.h" + +#define msgpack_pack_inline_func(name) \ + static inline void msgpack_pack ## name + +#define msgpack_pack_inline_func_cint(name) \ + static inline void msgpack_pack ## name + +#define msgpack_pack_user SV* + +#define msgpack_pack_append_buffer(user, buf, len) \ + sv_catpvn(user, (const char*)(buf), len); + +#include "msgpack/pack_template.h" + +#define _PACK_WRAPPER(t) msgpack_pack_##t +#define PACK_WRAPPER(t) _PACK_WRAPPER(t) + +// move to pack.c +static void _msgpack_pack_sv(SV* buf, SV* val) { + if (val==NULL) { + msgpack_pack_nil(buf); + return; + } + + switch (SvTYPE(val)) { + case SVt_NULL: + msgpack_pack_nil(buf); + break; + case SVt_IV: + if (SvIOK_UV(val)) { + msgpack_pack_uint32(buf, SvUV(val)); + } else { + PACK_WRAPPER(IVTYPE)(buf, SvIV(val)); + } + break; + case SVt_PVNV: + { + STRLEN len = 0; + char *pv = SvPV(val, len); + if (len == 1 && *pv == '1') { + msgpack_pack_true(buf); + } else if (len == 0 && *pv==0) { + msgpack_pack_false(buf); + } else { + msgpack_pack_nil(buf); + } + } + break; + case SVt_PV: + { + STRLEN len; + char * cval = SvPV(val, len); + msgpack_pack_raw(buf, len); + msgpack_pack_raw_body(buf, cval, len); + } + break; + case SVt_NV: + PACK_WRAPPER(NVTYPE)(buf, SvNV(val)); + break; + case SVt_PVAV: + { + AV* ary = (AV*)val; + int len = av_len(ary) + 1; + int i; + msgpack_pack_array(buf, len); + for (i=0; i is designed to support operation with Perl +installations back to 5.003, and has been tested up to 5.10.0. + +=head1 OPTIONS + +=head2 --help + +Display a brief usage summary. + +=head2 --version + +Display the version of F. + +=head2 --patch=I + +If this option is given, a single patch file will be created if +any changes are suggested. This requires a working diff program +to be installed on your system. + +=head2 --copy=I + +If this option is given, a copy of each file will be saved with +the given suffix that contains the suggested changes. This does +not require any external programs. Note that this does not +automagially add a dot between the original filename and the +suffix. If you want the dot, you have to include it in the option +argument. + +If neither C<--patch> or C<--copy> are given, the default is to +simply print the diffs for each file. This requires either +C or a C program to be installed. + +=head2 --diff=I + +Manually set the diff program and options to use. The default +is to use C, when installed, and output unified +context diffs. + +=head2 --compat-version=I + +Tell F to check for compatibility with the given +Perl version. The default is to check for compatibility with Perl +version 5.003. You can use this option to reduce the output +of F if you intend to be backward compatible only +down to a certain Perl version. + +=head2 --cplusplus + +Usually, F will detect C++ style comments and +replace them with C style comments for portability reasons. +Using this option instructs F to leave C++ +comments untouched. + +=head2 --quiet + +Be quiet. Don't print anything except fatal errors. + +=head2 --nodiag + +Don't output any diagnostic messages. Only portability +alerts will be printed. + +=head2 --nohints + +Don't output any hints. Hints often contain useful portability +notes. Warnings will still be displayed. + +=head2 --nochanges + +Don't suggest any changes. Only give diagnostic output and hints +unless these are also deactivated. + +=head2 --nofilter + +Don't filter the list of input files. By default, files not looking +like source code (i.e. not *.xs, *.c, *.cc, *.cpp or *.h) are skipped. + +=head2 --strip + +Strip all script and documentation functionality from F. +This reduces the size of F dramatically and may be useful +if you want to include F in smaller modules without +increasing their distribution size too much. + +The stripped F will have a C<--unstrip> option that allows +you to undo the stripping, but only if an appropriate C +module is installed. + +=head2 --list-provided + +Lists the API elements for which compatibility is provided by +F. Also lists if it must be explicitly requested, +if it has dependencies, and if there are hints or warnings for it. + +=head2 --list-unsupported + +Lists the API elements that are known not to be supported by +F and below which version of Perl they probably +won't be available or work. + +=head2 --api-info=I + +Show portability information for API elements matching I. +If I is surrounded by slashes, it is interpreted as a regular +expression. + +=head1 DESCRIPTION + +In order for a Perl extension (XS) module to be as portable as possible +across differing versions of Perl itself, certain steps need to be taken. + +=over 4 + +=item * + +Including this header is the first major one. This alone will give you +access to a large part of the Perl API that hasn't been available in +earlier Perl releases. Use + + perl ppport.h --list-provided + +to see which API elements are provided by ppport.h. + +=item * + +You should avoid using deprecated parts of the API. For example, using +global Perl variables without the C prefix is deprecated. Also, +some API functions used to have a C prefix. Using this form is +also deprecated. You can safely use the supported API, as F +will provide wrappers for older Perl versions. + +=item * + +If you use one of a few functions or variables that were not present in +earlier versions of Perl, and that can't be provided using a macro, you +have to explicitly request support for these functions by adding one or +more C<#define>s in your source code before the inclusion of F. + +These functions or variables will be marked C in the list shown +by C<--list-provided>. + +Depending on whether you module has a single or multiple files that +use such functions or variables, you want either C or global +variants. + +For a C function or variable (used only in a single source +file), use: + + #define NEED_function + #define NEED_variable + +For a global function or variable (used in multiple source files), +use: + + #define NEED_function_GLOBAL + #define NEED_variable_GLOBAL + +Note that you mustn't have more than one global request for the +same function or variable in your project. + + Function / Variable Static Request Global Request + ----------------------------------------------------------------------------------------- + PL_parser NEED_PL_parser NEED_PL_parser_GLOBAL + PL_signals NEED_PL_signals NEED_PL_signals_GLOBAL + eval_pv() NEED_eval_pv NEED_eval_pv_GLOBAL + grok_bin() NEED_grok_bin NEED_grok_bin_GLOBAL + grok_hex() NEED_grok_hex NEED_grok_hex_GLOBAL + grok_number() NEED_grok_number NEED_grok_number_GLOBAL + grok_numeric_radix() NEED_grok_numeric_radix NEED_grok_numeric_radix_GLOBAL + grok_oct() NEED_grok_oct NEED_grok_oct_GLOBAL + load_module() NEED_load_module NEED_load_module_GLOBAL + my_snprintf() NEED_my_snprintf NEED_my_snprintf_GLOBAL + my_sprintf() NEED_my_sprintf NEED_my_sprintf_GLOBAL + my_strlcat() NEED_my_strlcat NEED_my_strlcat_GLOBAL + my_strlcpy() NEED_my_strlcpy NEED_my_strlcpy_GLOBAL + newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL + newRV_noinc() NEED_newRV_noinc NEED_newRV_noinc_GLOBAL + newSVpvn_flags() NEED_newSVpvn_flags NEED_newSVpvn_flags_GLOBAL + newSVpvn_share() NEED_newSVpvn_share NEED_newSVpvn_share_GLOBAL + pv_display() NEED_pv_display NEED_pv_display_GLOBAL + pv_escape() NEED_pv_escape NEED_pv_escape_GLOBAL + pv_pretty() NEED_pv_pretty NEED_pv_pretty_GLOBAL + sv_2pv_flags() NEED_sv_2pv_flags NEED_sv_2pv_flags_GLOBAL + sv_2pvbyte() NEED_sv_2pvbyte NEED_sv_2pvbyte_GLOBAL + sv_catpvf_mg() NEED_sv_catpvf_mg NEED_sv_catpvf_mg_GLOBAL + sv_catpvf_mg_nocontext() NEED_sv_catpvf_mg_nocontext NEED_sv_catpvf_mg_nocontext_GLOBAL + sv_pvn_force_flags() NEED_sv_pvn_force_flags NEED_sv_pvn_force_flags_GLOBAL + sv_setpvf_mg() NEED_sv_setpvf_mg NEED_sv_setpvf_mg_GLOBAL + sv_setpvf_mg_nocontext() NEED_sv_setpvf_mg_nocontext NEED_sv_setpvf_mg_nocontext_GLOBAL + vload_module() NEED_vload_module NEED_vload_module_GLOBAL + vnewSVpvf() NEED_vnewSVpvf NEED_vnewSVpvf_GLOBAL + warner() NEED_warner NEED_warner_GLOBAL + +To avoid namespace conflicts, you can change the namespace of the +explicitly exported functions / variables using the C +macro. Just C<#define> the macro before including C: + + #define DPPP_NAMESPACE MyOwnNamespace_ + #include "ppport.h" + +The default namespace is C. + +=back + +The good thing is that most of the above can be checked by running +F on your source code. See the next section for +details. + +=head1 EXAMPLES + +To verify whether F is needed for your module, whether you +should make any changes to your code, and whether any special defines +should be used, F can be run as a Perl script to check your +source code. Simply say: + + perl ppport.h + +The result will usually be a list of patches suggesting changes +that should at least be acceptable, if not necessarily the most +efficient solution, or a fix for all possible problems. + +If you know that your XS module uses features only available in +newer Perl releases, if you're aware that it uses C++ comments, +and if you want all suggestions as a single patch file, you could +use something like this: + + perl ppport.h --compat-version=5.6.0 --cplusplus --patch=test.diff + +If you only want your code to be scanned without any suggestions +for changes, use: + + perl ppport.h --nochanges + +You can specify a different C program or options, using +the C<--diff> option: + + perl ppport.h --diff='diff -C 10' + +This would output context diffs with 10 lines of context. + +If you want to create patched copies of your files instead, use: + + perl ppport.h --copy=.new + +To display portability information for the C function, +use: + + perl ppport.h --api-info=newSVpvn + +Since the argument to C<--api-info> can be a regular expression, +you can use + + perl ppport.h --api-info=/_nomg$/ + +to display portability information for all C<_nomg> functions or + + perl ppport.h --api-info=/./ + +to display information for all known API elements. + +=head1 BUGS + +If this version of F is causing failure during +the compilation of this module, please check if newer versions +of either this module or C are available on CPAN +before sending a bug report. + +If F was generated using the latest version of +C and is causing failure of this module, please +file a bug report using the CPAN Request Tracker at L. + +Please include the following information: + +=over 4 + +=item 1. + +The complete output from running "perl -V" + +=item 2. + +This file. + +=item 3. + +The name and version of the module you were trying to build. + +=item 4. + +A full log of the build that failed. + +=item 5. + +Any other information that you think could be relevant. + +=back + +For the latest version of this code, please get the C +module from CPAN. + +=head1 COPYRIGHT + +Version 3.x, Copyright (c) 2004-2009, Marcus Holland-Moritz. + +Version 2.x, Copyright (C) 2001, Paul Marquess. + +Version 1.x, Copyright (C) 1999, Kenneth Albanowski. + +This program is free software; you can redistribute it and/or +modify it under the same terms as Perl itself. + +=head1 SEE ALSO + +See L. + +=cut + +use strict; + +# Disable broken TRIE-optimization +BEGIN { eval '${^RE_TRIE_MAXBUF} = -1' if $] >= 5.009004 && $] <= 5.009005 } + +my $VERSION = 3.17; + +my %opt = ( + quiet => 0, + diag => 1, + hints => 1, + changes => 1, + cplusplus => 0, + filter => 1, + strip => 0, + version => 0, +); + +my($ppport) = $0 =~ /([\w.]+)$/; +my $LF = '(?:\r\n|[\r\n])'; # line feed +my $HS = "[ \t]"; # horizontal whitespace + +# Never use C comments in this file! +my $ccs = '/'.'*'; +my $cce = '*'.'/'; +my $rccs = quotemeta $ccs; +my $rcce = quotemeta $cce; + +eval { + require Getopt::Long; + Getopt::Long::GetOptions(\%opt, qw( + help quiet diag! filter! hints! changes! cplusplus strip version + patch=s copy=s diff=s compat-version=s + list-provided list-unsupported api-info=s + )) or usage(); +}; + +if ($@ and grep /^-/, @ARGV) { + usage() if "@ARGV" =~ /^--?h(?:elp)?$/; + die "Getopt::Long not found. Please don't use any options.\n"; +} + +if ($opt{version}) { + print "This is $0 $VERSION.\n"; + exit 0; +} + +usage() if $opt{help}; +strip() if $opt{strip}; + +if (exists $opt{'compat-version'}) { + my($r,$v,$s) = eval { parse_version($opt{'compat-version'}) }; + if ($@) { + die "Invalid version number format: '$opt{'compat-version'}'\n"; + } + die "Only Perl 5 is supported\n" if $r != 5; + die "Invalid version number: $opt{'compat-version'}\n" if $v >= 1000 || $s >= 1000; + $opt{'compat-version'} = sprintf "%d.%03d%03d", $r, $v, $s; +} +else { + $opt{'compat-version'} = 5; +} + +my %API = map { /^(\w+)\|([^|]*)\|([^|]*)\|(\w*)$/ + ? ( $1 => { + ($2 ? ( base => $2 ) : ()), + ($3 ? ( todo => $3 ) : ()), + (index($4, 'v') >= 0 ? ( varargs => 1 ) : ()), + (index($4, 'p') >= 0 ? ( provided => 1 ) : ()), + (index($4, 'n') >= 0 ? ( nothxarg => 1 ) : ()), + } ) + : die "invalid spec: $_" } qw( +AvFILLp|5.004050||p +AvFILL||| +CLASS|||n +CPERLscope|5.005000||p +CX_CURPAD_SAVE||| +CX_CURPAD_SV||| +CopFILEAV|5.006000||p +CopFILEGV_set|5.006000||p +CopFILEGV|5.006000||p +CopFILESV|5.006000||p +CopFILE_set|5.006000||p +CopFILE|5.006000||p +CopSTASHPV_set|5.006000||p +CopSTASHPV|5.006000||p +CopSTASH_eq|5.006000||p +CopSTASH_set|5.006000||p +CopSTASH|5.006000||p +CopyD|5.009002||p +Copy||| +CvPADLIST||| +CvSTASH||| +CvWEAKOUTSIDE||| +DEFSV_set|5.011000||p +DEFSV|5.004050||p +END_EXTERN_C|5.005000||p +ENTER||| +ERRSV|5.004050||p +EXTEND||| +EXTERN_C|5.005000||p +F0convert|||n +FREETMPS||| +GIMME_V||5.004000|n +GIMME|||n +GROK_NUMERIC_RADIX|5.007002||p +G_ARRAY||| +G_DISCARD||| +G_EVAL||| +G_METHOD|5.006001||p +G_NOARGS||| +G_SCALAR||| +G_VOID||5.004000| +GetVars||| +GvSV||| +Gv_AMupdate||| +HEf_SVKEY||5.004000| +HeHASH||5.004000| +HeKEY||5.004000| +HeKLEN||5.004000| +HePV||5.004000| +HeSVKEY_force||5.004000| +HeSVKEY_set||5.004000| +HeSVKEY||5.004000| +HeUTF8||5.011000| +HeVAL||5.004000| +HvNAME||| +INT2PTR|5.006000||p +IN_LOCALE_COMPILETIME|5.007002||p +IN_LOCALE_RUNTIME|5.007002||p +IN_LOCALE|5.007002||p +IN_PERL_COMPILETIME|5.008001||p +IS_NUMBER_GREATER_THAN_UV_MAX|5.007002||p +IS_NUMBER_INFINITY|5.007002||p +IS_NUMBER_IN_UV|5.007002||p +IS_NUMBER_NAN|5.007003||p +IS_NUMBER_NEG|5.007002||p +IS_NUMBER_NOT_INT|5.007002||p +IVSIZE|5.006000||p +IVTYPE|5.006000||p +IVdf|5.006000||p +LEAVE||| +LVRET||| +MARK||| +MULTICALL||5.011000| +MY_CXT_CLONE|5.009002||p +MY_CXT_INIT|5.007003||p +MY_CXT|5.007003||p +MoveD|5.009002||p +Move||| +NOOP|5.005000||p +NUM2PTR|5.006000||p +NVTYPE|5.006000||p +NVef|5.006001||p +NVff|5.006001||p +NVgf|5.006001||p +Newxc|5.009003||p +Newxz|5.009003||p +Newx|5.009003||p +Nullav||| +Nullch||| +Nullcv||| +Nullhv||| +Nullsv||| +ORIGMARK||| +PAD_BASE_SV||| +PAD_CLONE_VARS||| +PAD_COMPNAME_FLAGS||| +PAD_COMPNAME_GEN_set||| +PAD_COMPNAME_GEN||| +PAD_COMPNAME_OURSTASH||| +PAD_COMPNAME_PV||| +PAD_COMPNAME_TYPE||| +PAD_DUP||| +PAD_RESTORE_LOCAL||| +PAD_SAVE_LOCAL||| +PAD_SAVE_SETNULLPAD||| +PAD_SETSV||| +PAD_SET_CUR_NOSAVE||| +PAD_SET_CUR||| +PAD_SVl||| +PAD_SV||| +PERLIO_FUNCS_CAST|5.009003||p +PERLIO_FUNCS_DECL|5.009003||p +PERL_ABS|5.008001||p +PERL_BCDVERSION|5.011000||p +PERL_GCC_BRACE_GROUPS_FORBIDDEN|5.008001||p +PERL_HASH|5.004000||p +PERL_INT_MAX|5.004000||p +PERL_INT_MIN|5.004000||p +PERL_LONG_MAX|5.004000||p +PERL_LONG_MIN|5.004000||p +PERL_MAGIC_arylen|5.007002||p +PERL_MAGIC_backref|5.007002||p +PERL_MAGIC_bm|5.007002||p +PERL_MAGIC_collxfrm|5.007002||p +PERL_MAGIC_dbfile|5.007002||p +PERL_MAGIC_dbline|5.007002||p +PERL_MAGIC_defelem|5.007002||p +PERL_MAGIC_envelem|5.007002||p +PERL_MAGIC_env|5.007002||p +PERL_MAGIC_ext|5.007002||p +PERL_MAGIC_fm|5.007002||p +PERL_MAGIC_glob|5.011000||p +PERL_MAGIC_isaelem|5.007002||p +PERL_MAGIC_isa|5.007002||p +PERL_MAGIC_mutex|5.011000||p +PERL_MAGIC_nkeys|5.007002||p +PERL_MAGIC_overload_elem|5.007002||p +PERL_MAGIC_overload_table|5.007002||p +PERL_MAGIC_overload|5.007002||p +PERL_MAGIC_pos|5.007002||p +PERL_MAGIC_qr|5.007002||p +PERL_MAGIC_regdata|5.007002||p +PERL_MAGIC_regdatum|5.007002||p +PERL_MAGIC_regex_global|5.007002||p +PERL_MAGIC_shared_scalar|5.007003||p +PERL_MAGIC_shared|5.007003||p +PERL_MAGIC_sigelem|5.007002||p +PERL_MAGIC_sig|5.007002||p +PERL_MAGIC_substr|5.007002||p +PERL_MAGIC_sv|5.007002||p +PERL_MAGIC_taint|5.007002||p +PERL_MAGIC_tiedelem|5.007002||p +PERL_MAGIC_tiedscalar|5.007002||p +PERL_MAGIC_tied|5.007002||p +PERL_MAGIC_utf8|5.008001||p +PERL_MAGIC_uvar_elem|5.007003||p +PERL_MAGIC_uvar|5.007002||p +PERL_MAGIC_vec|5.007002||p +PERL_MAGIC_vstring|5.008001||p +PERL_PV_ESCAPE_ALL|5.009004||p +PERL_PV_ESCAPE_FIRSTCHAR|5.009004||p +PERL_PV_ESCAPE_NOBACKSLASH|5.009004||p +PERL_PV_ESCAPE_NOCLEAR|5.009004||p +PERL_PV_ESCAPE_QUOTE|5.009004||p +PERL_PV_ESCAPE_RE|5.009005||p +PERL_PV_ESCAPE_UNI_DETECT|5.009004||p +PERL_PV_ESCAPE_UNI|5.009004||p +PERL_PV_PRETTY_DUMP|5.009004||p +PERL_PV_PRETTY_ELLIPSES|5.010000||p +PERL_PV_PRETTY_LTGT|5.009004||p +PERL_PV_PRETTY_NOCLEAR|5.010000||p +PERL_PV_PRETTY_QUOTE|5.009004||p +PERL_PV_PRETTY_REGPROP|5.009004||p +PERL_QUAD_MAX|5.004000||p +PERL_QUAD_MIN|5.004000||p +PERL_REVISION|5.006000||p +PERL_SCAN_ALLOW_UNDERSCORES|5.007003||p +PERL_SCAN_DISALLOW_PREFIX|5.007003||p +PERL_SCAN_GREATER_THAN_UV_MAX|5.007003||p +PERL_SCAN_SILENT_ILLDIGIT|5.008001||p +PERL_SHORT_MAX|5.004000||p +PERL_SHORT_MIN|5.004000||p +PERL_SIGNALS_UNSAFE_FLAG|5.008001||p +PERL_SUBVERSION|5.006000||p +PERL_UCHAR_MAX|5.004000||p +PERL_UCHAR_MIN|5.004000||p +PERL_UINT_MAX|5.004000||p +PERL_UINT_MIN|5.004000||p +PERL_ULONG_MAX|5.004000||p +PERL_ULONG_MIN|5.004000||p +PERL_UNUSED_ARG|5.009003||p +PERL_UNUSED_CONTEXT|5.009004||p +PERL_UNUSED_DECL|5.007002||p +PERL_UNUSED_VAR|5.007002||p +PERL_UQUAD_MAX|5.004000||p +PERL_UQUAD_MIN|5.004000||p +PERL_USE_GCC_BRACE_GROUPS|5.009004||p +PERL_USHORT_MAX|5.004000||p +PERL_USHORT_MIN|5.004000||p +PERL_VERSION|5.006000||p +PL_DBsignal|5.005000||p +PL_DBsingle|||pn +PL_DBsub|||pn +PL_DBtrace|||pn +PL_Sv|5.005000||p +PL_bufend|5.011000||p +PL_bufptr|5.011000||p +PL_compiling|5.004050||p +PL_copline|5.011000||p +PL_curcop|5.004050||p +PL_curstash|5.004050||p +PL_debstash|5.004050||p +PL_defgv|5.004050||p +PL_diehook|5.004050||p +PL_dirty|5.004050||p +PL_dowarn|||pn +PL_errgv|5.004050||p +PL_expect|5.011000||p +PL_hexdigit|5.005000||p +PL_hints|5.005000||p +PL_last_in_gv|||n +PL_laststatval|5.005000||p +PL_lex_state|5.011000||p +PL_lex_stuff|5.011000||p +PL_linestr|5.011000||p +PL_modglobal||5.005000|n +PL_na|5.004050||pn +PL_no_modify|5.006000||p +PL_ofsgv|||n +PL_parser|5.009005||p +PL_perl_destruct_level|5.004050||p +PL_perldb|5.004050||p +PL_ppaddr|5.006000||p +PL_rsfp_filters|5.004050||p +PL_rsfp|5.004050||p +PL_rs|||n +PL_signals|5.008001||p +PL_stack_base|5.004050||p +PL_stack_sp|5.004050||p +PL_statcache|5.005000||p +PL_stdingv|5.004050||p +PL_sv_arenaroot|5.004050||p +PL_sv_no|5.004050||pn +PL_sv_undef|5.004050||pn +PL_sv_yes|5.004050||pn +PL_tainted|5.004050||p +PL_tainting|5.004050||p +PL_tokenbuf|5.011000||p +POP_MULTICALL||5.011000| +POPi|||n +POPl|||n +POPn|||n +POPpbytex||5.007001|n +POPpx||5.005030|n +POPp|||n +POPs|||n +PTR2IV|5.006000||p +PTR2NV|5.006000||p +PTR2UV|5.006000||p +PTR2nat|5.009003||p +PTR2ul|5.007001||p +PTRV|5.006000||p +PUSHMARK||| +PUSH_MULTICALL||5.011000| +PUSHi||| +PUSHmortal|5.009002||p +PUSHn||| +PUSHp||| +PUSHs||| +PUSHu|5.004000||p +PUTBACK||| +PerlIO_clearerr||5.007003| +PerlIO_close||5.007003| +PerlIO_context_layers||5.009004| +PerlIO_eof||5.007003| +PerlIO_error||5.007003| +PerlIO_fileno||5.007003| +PerlIO_fill||5.007003| +PerlIO_flush||5.007003| +PerlIO_get_base||5.007003| +PerlIO_get_bufsiz||5.007003| +PerlIO_get_cnt||5.007003| +PerlIO_get_ptr||5.007003| +PerlIO_read||5.007003| +PerlIO_seek||5.007003| +PerlIO_set_cnt||5.007003| +PerlIO_set_ptrcnt||5.007003| +PerlIO_setlinebuf||5.007003| +PerlIO_stderr||5.007003| +PerlIO_stdin||5.007003| +PerlIO_stdout||5.007003| +PerlIO_tell||5.007003| +PerlIO_unread||5.007003| +PerlIO_write||5.007003| +Perl_signbit||5.009005|n +PoisonFree|5.009004||p +PoisonNew|5.009004||p +PoisonWith|5.009004||p +Poison|5.008000||p +RETVAL|||n +Renewc||| +Renew||| +SAVECLEARSV||| +SAVECOMPPAD||| +SAVEPADSV||| +SAVETMPS||| +SAVE_DEFSV|5.004050||p +SPAGAIN||| +SP||| +START_EXTERN_C|5.005000||p +START_MY_CXT|5.007003||p +STMT_END|||p +STMT_START|||p +STR_WITH_LEN|5.009003||p +ST||| +SV_CONST_RETURN|5.009003||p +SV_COW_DROP_PV|5.008001||p +SV_COW_SHARED_HASH_KEYS|5.009005||p +SV_GMAGIC|5.007002||p +SV_HAS_TRAILING_NUL|5.009004||p +SV_IMMEDIATE_UNREF|5.007001||p +SV_MUTABLE_RETURN|5.009003||p +SV_NOSTEAL|5.009002||p +SV_SMAGIC|5.009003||p +SV_UTF8_NO_ENCODING|5.008001||p +SVf_UTF8|5.006000||p +SVf|5.006000||p +SVt_IV||| +SVt_NV||| +SVt_PVAV||| +SVt_PVCV||| +SVt_PVHV||| +SVt_PVMG||| +SVt_PV||| +Safefree||| +Slab_Alloc||| +Slab_Free||| +Slab_to_rw||| +StructCopy||| +SvCUR_set||| +SvCUR||| +SvEND||| +SvGAMAGIC||5.006001| +SvGETMAGIC|5.004050||p +SvGROW||| +SvIOK_UV||5.006000| +SvIOK_notUV||5.006000| +SvIOK_off||| +SvIOK_only_UV||5.006000| +SvIOK_only||| +SvIOK_on||| +SvIOKp||| +SvIOK||| +SvIVX||| +SvIV_nomg|5.009001||p +SvIV_set||| +SvIVx||| +SvIV||| +SvIsCOW_shared_hash||5.008003| +SvIsCOW||5.008003| +SvLEN_set||| +SvLEN||| +SvLOCK||5.007003| +SvMAGIC_set|5.009003||p +SvNIOK_off||| +SvNIOKp||| +SvNIOK||| +SvNOK_off||| +SvNOK_only||| +SvNOK_on||| +SvNOKp||| +SvNOK||| +SvNVX||| +SvNV_set||| +SvNVx||| +SvNV||| +SvOK||| +SvOOK_offset||5.011000| +SvOOK||| +SvPOK_off||| +SvPOK_only_UTF8||5.006000| +SvPOK_only||| +SvPOK_on||| +SvPOKp||| +SvPOK||| +SvPVX_const|5.009003||p +SvPVX_mutable|5.009003||p +SvPVX||| +SvPV_const|5.009003||p +SvPV_flags_const_nolen|5.009003||p +SvPV_flags_const|5.009003||p +SvPV_flags_mutable|5.009003||p +SvPV_flags|5.007002||p +SvPV_force_flags_mutable|5.009003||p +SvPV_force_flags_nolen|5.009003||p +SvPV_force_flags|5.007002||p +SvPV_force_mutable|5.009003||p +SvPV_force_nolen|5.009003||p +SvPV_force_nomg_nolen|5.009003||p +SvPV_force_nomg|5.007002||p +SvPV_force|||p +SvPV_mutable|5.009003||p +SvPV_nolen_const|5.009003||p +SvPV_nolen|5.006000||p +SvPV_nomg_const_nolen|5.009003||p +SvPV_nomg_const|5.009003||p +SvPV_nomg|5.007002||p +SvPV_renew|5.009003||p +SvPV_set||| +SvPVbyte_force||5.009002| +SvPVbyte_nolen||5.006000| +SvPVbytex_force||5.006000| +SvPVbytex||5.006000| +SvPVbyte|5.006000||p +SvPVutf8_force||5.006000| +SvPVutf8_nolen||5.006000| +SvPVutf8x_force||5.006000| +SvPVutf8x||5.006000| +SvPVutf8||5.006000| +SvPVx||| +SvPV||| +SvREFCNT_dec||| +SvREFCNT_inc_NN|5.009004||p +SvREFCNT_inc_simple_NN|5.009004||p +SvREFCNT_inc_simple_void_NN|5.009004||p +SvREFCNT_inc_simple_void|5.009004||p +SvREFCNT_inc_simple|5.009004||p +SvREFCNT_inc_void_NN|5.009004||p +SvREFCNT_inc_void|5.009004||p +SvREFCNT_inc|||p +SvREFCNT||| +SvROK_off||| +SvROK_on||| +SvROK||| +SvRV_set|5.009003||p +SvRV||| +SvRXOK||5.009005| +SvRX||5.009005| +SvSETMAGIC||| +SvSHARED_HASH|5.009003||p +SvSHARE||5.007003| +SvSTASH_set|5.009003||p +SvSTASH||| +SvSetMagicSV_nosteal||5.004000| +SvSetMagicSV||5.004000| +SvSetSV_nosteal||5.004000| +SvSetSV||| +SvTAINTED_off||5.004000| +SvTAINTED_on||5.004000| +SvTAINTED||5.004000| +SvTAINT||| +SvTRUE||| +SvTYPE||| +SvUNLOCK||5.007003| +SvUOK|5.007001|5.006000|p +SvUPGRADE||| +SvUTF8_off||5.006000| +SvUTF8_on||5.006000| +SvUTF8||5.006000| +SvUVXx|5.004000||p +SvUVX|5.004000||p +SvUV_nomg|5.009001||p +SvUV_set|5.009003||p +SvUVx|5.004000||p +SvUV|5.004000||p +SvVOK||5.008001| +SvVSTRING_mg|5.009004||p +THIS|||n +UNDERBAR|5.009002||p +UTF8_MAXBYTES|5.009002||p +UVSIZE|5.006000||p +UVTYPE|5.006000||p +UVXf|5.007001||p +UVof|5.006000||p +UVuf|5.006000||p +UVxf|5.006000||p +WARN_ALL|5.006000||p +WARN_AMBIGUOUS|5.006000||p +WARN_ASSERTIONS|5.011000||p +WARN_BAREWORD|5.006000||p +WARN_CLOSED|5.006000||p +WARN_CLOSURE|5.006000||p +WARN_DEBUGGING|5.006000||p +WARN_DEPRECATED|5.006000||p +WARN_DIGIT|5.006000||p +WARN_EXEC|5.006000||p +WARN_EXITING|5.006000||p +WARN_GLOB|5.006000||p +WARN_INPLACE|5.006000||p +WARN_INTERNAL|5.006000||p +WARN_IO|5.006000||p +WARN_LAYER|5.008000||p +WARN_MALLOC|5.006000||p +WARN_MISC|5.006000||p +WARN_NEWLINE|5.006000||p +WARN_NUMERIC|5.006000||p +WARN_ONCE|5.006000||p +WARN_OVERFLOW|5.006000||p +WARN_PACK|5.006000||p +WARN_PARENTHESIS|5.006000||p +WARN_PIPE|5.006000||p +WARN_PORTABLE|5.006000||p +WARN_PRECEDENCE|5.006000||p +WARN_PRINTF|5.006000||p +WARN_PROTOTYPE|5.006000||p +WARN_QW|5.006000||p +WARN_RECURSION|5.006000||p +WARN_REDEFINE|5.006000||p +WARN_REGEXP|5.006000||p +WARN_RESERVED|5.006000||p +WARN_SEMICOLON|5.006000||p +WARN_SEVERE|5.006000||p +WARN_SIGNAL|5.006000||p +WARN_SUBSTR|5.006000||p +WARN_SYNTAX|5.006000||p +WARN_TAINT|5.006000||p +WARN_THREADS|5.008000||p +WARN_UNINITIALIZED|5.006000||p +WARN_UNOPENED|5.006000||p +WARN_UNPACK|5.006000||p +WARN_UNTIE|5.006000||p +WARN_UTF8|5.006000||p +WARN_VOID|5.006000||p +XCPT_CATCH|5.009002||p +XCPT_RETHROW|5.009002||p +XCPT_TRY_END|5.009002||p +XCPT_TRY_START|5.009002||p +XPUSHi||| +XPUSHmortal|5.009002||p +XPUSHn||| +XPUSHp||| +XPUSHs||| +XPUSHu|5.004000||p +XSRETURN_EMPTY||| +XSRETURN_IV||| +XSRETURN_NO||| +XSRETURN_NV||| +XSRETURN_PV||| +XSRETURN_UNDEF||| +XSRETURN_UV|5.008001||p +XSRETURN_YES||| +XSRETURN|||p +XST_mIV||| +XST_mNO||| +XST_mNV||| +XST_mPV||| +XST_mUNDEF||| +XST_mUV|5.008001||p +XST_mYES||| +XS_VERSION_BOOTCHECK||| +XS_VERSION||| +XSprePUSH|5.006000||p +XS||| +ZeroD|5.009002||p +Zero||| +_aMY_CXT|5.007003||p +_pMY_CXT|5.007003||p +aMY_CXT_|5.007003||p +aMY_CXT|5.007003||p +aTHXR_|5.011000||p +aTHXR|5.011000||p +aTHX_|5.006000||p +aTHX|5.006000||p +add_data|||n +addmad||| +allocmy||| +amagic_call||| +amagic_cmp_locale||| +amagic_cmp||| +amagic_i_ncmp||| +amagic_ncmp||| +any_dup||| +ao||| +append_elem||| +append_list||| +append_madprops||| +apply_attrs_my||| +apply_attrs_string||5.006001| +apply_attrs||| +apply||| +atfork_lock||5.007003|n +atfork_unlock||5.007003|n +av_arylen_p||5.009003| +av_clear||| +av_create_and_push||5.009005| +av_create_and_unshift_one||5.009005| +av_delete||5.006000| +av_exists||5.006000| +av_extend||| +av_fetch||| +av_fill||| +av_iter_p||5.011000| +av_len||| +av_make||| +av_pop||| +av_push||| +av_reify||| +av_shift||| +av_store||| +av_undef||| +av_unshift||| +ax|||n +bad_type||| +bind_match||| +block_end||| +block_gimme||5.004000| +block_start||| +boolSV|5.004000||p +boot_core_PerlIO||| +boot_core_UNIVERSAL||| +boot_core_mro||| +boot_core_xsutils||| +bytes_from_utf8||5.007001| +bytes_to_uni|||n +bytes_to_utf8||5.006001| +call_argv|5.006000||p +call_atexit||5.006000| +call_list||5.004000| +call_method|5.006000||p +call_pv|5.006000||p +call_sv|5.006000||p +calloc||5.007002|n +cando||| +cast_i32||5.006000| +cast_iv||5.006000| +cast_ulong||5.006000| +cast_uv||5.006000| +check_type_and_open||| +check_uni||| +checkcomma||| +checkposixcc||| +ckWARN|5.006000||p +ck_anoncode||| +ck_bitop||| +ck_concat||| +ck_defined||| +ck_delete||| +ck_die||| +ck_each||| +ck_eof||| +ck_eval||| +ck_exec||| +ck_exists||| +ck_exit||| +ck_ftst||| +ck_fun||| +ck_glob||| +ck_grep||| +ck_index||| +ck_join||| +ck_lfun||| +ck_listiob||| +ck_match||| +ck_method||| +ck_null||| +ck_open||| +ck_readline||| +ck_repeat||| +ck_require||| +ck_return||| +ck_rfun||| +ck_rvconst||| +ck_sassign||| +ck_select||| +ck_shift||| +ck_sort||| +ck_spair||| +ck_split||| +ck_subr||| +ck_substr||| +ck_svconst||| +ck_trunc||| +ck_unpack||| +ckwarn_d||5.009003| +ckwarn||5.009003| +cl_and|||n +cl_anything|||n +cl_init_zero|||n +cl_init|||n +cl_is_anything|||n +cl_or|||n +clear_placeholders||| +closest_cop||| +convert||| +cop_free||| +cr_textfilter||| +create_eval_scope||| +croak_nocontext|||vn +croak_xs_usage||5.011000| +croak|||v +csighandler||5.009003|n +curmad||| +custom_op_desc||5.007003| +custom_op_name||5.007003| +cv_ckproto_len||| +cv_clone||| +cv_const_sv||5.004000| +cv_dump||| +cv_undef||| +cx_dump||5.005000| +cx_dup||| +cxinc||| +dAXMARK|5.009003||p +dAX|5.007002||p +dITEMS|5.007002||p +dMARK||| +dMULTICALL||5.009003| +dMY_CXT_SV|5.007003||p +dMY_CXT|5.007003||p +dNOOP|5.006000||p +dORIGMARK||| +dSP||| +dTHR|5.004050||p +dTHXR|5.011000||p +dTHXa|5.006000||p +dTHXoa|5.006000||p +dTHX|5.006000||p +dUNDERBAR|5.009002||p +dVAR|5.009003||p +dXCPT|5.009002||p +dXSARGS||| +dXSI32||| +dXSTARG|5.006000||p +deb_curcv||| +deb_nocontext|||vn +deb_stack_all||| +deb_stack_n||| +debop||5.005000| +debprofdump||5.005000| +debprof||| +debstackptrs||5.007003| +debstack||5.007003| +debug_start_match||| +deb||5.007003|v +del_sv||| +delete_eval_scope||| +delimcpy||5.004000| +deprecate_old||| +deprecate||| +despatch_signals||5.007001| +destroy_matcher||| +die_nocontext|||vn +die_where||| +die|||v +dirp_dup||| +div128||| +djSP||| +do_aexec5||| +do_aexec||| +do_aspawn||| +do_binmode||5.004050| +do_chomp||| +do_chop||| +do_close||| +do_dump_pad||| +do_eof||| +do_exec3||| +do_execfree||| +do_exec||| +do_gv_dump||5.006000| +do_gvgv_dump||5.006000| +do_hv_dump||5.006000| +do_ipcctl||| +do_ipcget||| +do_join||| +do_kv||| +do_magic_dump||5.006000| +do_msgrcv||| +do_msgsnd||| +do_oddball||| +do_op_dump||5.006000| +do_op_xmldump||| +do_open9||5.006000| +do_openn||5.007001| +do_open||5.004000| +do_pmop_dump||5.006000| +do_pmop_xmldump||| +do_print||| +do_readline||| +do_seek||| +do_semop||| +do_shmio||| +do_smartmatch||| +do_spawn_nowait||| +do_spawn||| +do_sprintf||| +do_sv_dump||5.006000| +do_sysseek||| +do_tell||| +do_trans_complex_utf8||| +do_trans_complex||| +do_trans_count_utf8||| +do_trans_count||| +do_trans_simple_utf8||| +do_trans_simple||| +do_trans||| +do_vecget||| +do_vecset||| +do_vop||| +docatch||| +doeval||| +dofile||| +dofindlabel||| +doform||| +doing_taint||5.008001|n +dooneliner||| +doopen_pm||| +doparseform||| +dopoptoeval||| +dopoptogiven||| +dopoptolabel||| +dopoptoloop||| +dopoptosub_at||| +dopoptowhen||| +doref||5.009003| +dounwind||| +dowantarray||| +dump_all||5.006000| +dump_eval||5.006000| +dump_exec_pos||| +dump_fds||| +dump_form||5.006000| +dump_indent||5.006000|v +dump_mstats||| +dump_packsubs||5.006000| +dump_sub||5.006000| +dump_sv_child||| +dump_trie_interim_list||| +dump_trie_interim_table||| +dump_trie||| +dump_vindent||5.006000| +dumpuntil||| +dup_attrlist||| +emulate_cop_io||| +eval_pv|5.006000||p +eval_sv|5.006000||p +exec_failed||| +expect_number||| +fbm_compile||5.005000| +fbm_instr||5.005000| +feature_is_enabled||| +fetch_cop_label||5.011000| +filter_add||| +filter_del||| +filter_gets||| +filter_read||| +find_and_forget_pmops||| +find_array_subscript||| +find_beginning||| +find_byclass||| +find_hash_subscript||| +find_in_my_stash||| +find_runcv||5.008001| +find_rundefsvoffset||5.009002| +find_script||| +find_uninit_var||| +first_symbol|||n +fold_constants||| +forbid_setid||| +force_ident||| +force_list||| +force_next||| +force_version||| +force_word||| +forget_pmop||| +form_nocontext|||vn +form||5.004000|v +fp_dup||| +fprintf_nocontext|||vn +free_global_struct||| +free_tied_hv_pool||| +free_tmps||| +gen_constant_list||| +get_arena||| +get_aux_mg||| +get_av|5.006000||p +get_context||5.006000|n +get_cvn_flags||5.009005| +get_cv|5.006000||p +get_db_sub||| +get_debug_opts||| +get_hash_seed||| +get_hv|5.006000||p +get_isa_hash||| +get_mstats||| +get_no_modify||| +get_num||| +get_op_descs||5.005000| +get_op_names||5.005000| +get_opargs||| +get_ppaddr||5.006000| +get_re_arg||| +get_sv|5.006000||p +get_vtbl||5.005030| +getcwd_sv||5.007002| +getenv_len||| +glob_2number||| +glob_2pv||| +glob_assign_glob||| +glob_assign_ref||| +gp_dup||| +gp_free||| +gp_ref||| +grok_bin|5.007003||p +grok_hex|5.007003||p +grok_number|5.007002||p +grok_numeric_radix|5.007002||p +grok_oct|5.007003||p +group_end||| +gv_AVadd||| +gv_HVadd||| +gv_IOadd||| +gv_SVadd||| +gv_autoload4||5.004000| +gv_check||| +gv_const_sv||5.009003| +gv_dump||5.006000| +gv_efullname3||5.004000| +gv_efullname4||5.006001| +gv_efullname||| +gv_ename||| +gv_fetchfile_flags||5.009005| +gv_fetchfile||| +gv_fetchmeth_autoload||5.007003| +gv_fetchmethod_autoload||5.004000| +gv_fetchmethod_flags||5.011000| +gv_fetchmethod||| +gv_fetchmeth||| +gv_fetchpvn_flags||5.009002| +gv_fetchpv||| +gv_fetchsv||5.009002| +gv_fullname3||5.004000| +gv_fullname4||5.006001| +gv_fullname||| +gv_get_super_pkg||| +gv_handler||5.007001| +gv_init_sv||| +gv_init||| +gv_name_set||5.009004| +gv_stashpvn|5.004000||p +gv_stashpvs||5.009003| +gv_stashpv||| +gv_stashsv||| +he_dup||| +hek_dup||| +hfreeentries||| +hsplit||| +hv_assert||5.011000| +hv_auxinit|||n +hv_backreferences_p||| +hv_clear_placeholders||5.009001| +hv_clear||| +hv_common_key_len||5.010000| +hv_common||5.010000| +hv_copy_hints_hv||| +hv_delayfree_ent||5.004000| +hv_delete_common||| +hv_delete_ent||5.004000| +hv_delete||| +hv_eiter_p||5.009003| +hv_eiter_set||5.009003| +hv_exists_ent||5.004000| +hv_exists||| +hv_fetch_ent||5.004000| +hv_fetchs|5.009003||p +hv_fetch||| +hv_free_ent||5.004000| +hv_iterinit||| +hv_iterkeysv||5.004000| +hv_iterkey||| +hv_iternext_flags||5.008000| +hv_iternextsv||| +hv_iternext||| +hv_iterval||| +hv_kill_backrefs||| +hv_ksplit||5.004000| +hv_magic_check|||n +hv_magic||| +hv_name_set||5.009003| +hv_notallowed||| +hv_placeholders_get||5.009003| +hv_placeholders_p||5.009003| +hv_placeholders_set||5.009003| +hv_riter_p||5.009003| +hv_riter_set||5.009003| +hv_scalar||5.009001| +hv_store_ent||5.004000| +hv_store_flags||5.008000| +hv_stores|5.009004||p +hv_store||| +hv_undef||| +ibcmp_locale||5.004000| +ibcmp_utf8||5.007003| +ibcmp||| +incline||| +incpush_if_exists||| +incpush_use_sep||| +incpush||| +ingroup||| +init_argv_symbols||| +init_debugger||| +init_global_struct||| +init_i18nl10n||5.006000| +init_i18nl14n||5.006000| +init_ids||| +init_interp||| +init_main_stash||| +init_perllib||| +init_postdump_symbols||| +init_predump_symbols||| +init_stacks||5.005000| +init_tm||5.007002| +instr||| +intro_my||| +intuit_method||| +intuit_more||| +invert||| +io_close||| +isALNUMC|5.006000||p +isALNUM||| +isALPHA||| +isASCII|5.006000||p +isBLANK|5.006001||p +isCNTRL|5.006000||p +isDIGIT||| +isGRAPH|5.006000||p +isLOWER||| +isPRINT|5.004000||p +isPSXSPC|5.006001||p +isPUNCT|5.006000||p +isSPACE||| +isUPPER||| +isXDIGIT|5.006000||p +is_an_int||| +is_gv_magical_sv||| +is_handle_constructor|||n +is_list_assignment||| +is_lvalue_sub||5.007001| +is_uni_alnum_lc||5.006000| +is_uni_alnumc_lc||5.006000| +is_uni_alnumc||5.006000| +is_uni_alnum||5.006000| +is_uni_alpha_lc||5.006000| +is_uni_alpha||5.006000| +is_uni_ascii_lc||5.006000| +is_uni_ascii||5.006000| +is_uni_cntrl_lc||5.006000| +is_uni_cntrl||5.006000| +is_uni_digit_lc||5.006000| +is_uni_digit||5.006000| +is_uni_graph_lc||5.006000| +is_uni_graph||5.006000| +is_uni_idfirst_lc||5.006000| +is_uni_idfirst||5.006000| +is_uni_lower_lc||5.006000| +is_uni_lower||5.006000| +is_uni_print_lc||5.006000| +is_uni_print||5.006000| +is_uni_punct_lc||5.006000| +is_uni_punct||5.006000| +is_uni_space_lc||5.006000| +is_uni_space||5.006000| +is_uni_upper_lc||5.006000| +is_uni_upper||5.006000| +is_uni_xdigit_lc||5.006000| +is_uni_xdigit||5.006000| +is_utf8_alnumc||5.006000| +is_utf8_alnum||5.006000| +is_utf8_alpha||5.006000| +is_utf8_ascii||5.006000| +is_utf8_char_slow|||n +is_utf8_char||5.006000| +is_utf8_cntrl||5.006000| +is_utf8_common||| +is_utf8_digit||5.006000| +is_utf8_graph||5.006000| +is_utf8_idcont||5.008000| +is_utf8_idfirst||5.006000| +is_utf8_lower||5.006000| +is_utf8_mark||5.006000| +is_utf8_print||5.006000| +is_utf8_punct||5.006000| +is_utf8_space||5.006000| +is_utf8_string_loclen||5.009003| +is_utf8_string_loc||5.008001| +is_utf8_string||5.006001| +is_utf8_upper||5.006000| +is_utf8_xdigit||5.006000| +isa_lookup||| +items|||n +ix|||n +jmaybe||| +join_exact||| +keyword||| +leave_scope||| +lex_end||| +lex_start||| +linklist||| +listkids||| +list||| +load_module_nocontext|||vn +load_module|5.006000||pv +localize||| +looks_like_bool||| +looks_like_number||| +lop||| +mPUSHi|5.009002||p +mPUSHn|5.009002||p +mPUSHp|5.009002||p +mPUSHs|5.011000||p +mPUSHu|5.009002||p +mXPUSHi|5.009002||p +mXPUSHn|5.009002||p +mXPUSHp|5.009002||p +mXPUSHs|5.011000||p +mXPUSHu|5.009002||p +mad_free||| +madlex||| +madparse||| +magic_clear_all_env||| +magic_clearenv||| +magic_clearhint||| +magic_clearisa||| +magic_clearpack||| +magic_clearsig||| +magic_dump||5.006000| +magic_existspack||| +magic_freearylen_p||| +magic_freeovrld||| +magic_getarylen||| +magic_getdefelem||| +magic_getnkeys||| +magic_getpack||| +magic_getpos||| +magic_getsig||| +magic_getsubstr||| +magic_gettaint||| +magic_getuvar||| +magic_getvec||| +magic_get||| +magic_killbackrefs||| +magic_len||| +magic_methcall||| +magic_methpack||| +magic_nextpack||| +magic_regdata_cnt||| +magic_regdatum_get||| +magic_regdatum_set||| +magic_scalarpack||| +magic_set_all_env||| +magic_setamagic||| +magic_setarylen||| +magic_setcollxfrm||| +magic_setdbline||| +magic_setdefelem||| +magic_setenv||| +magic_sethint||| +magic_setisa||| +magic_setmglob||| +magic_setnkeys||| +magic_setpack||| +magic_setpos||| +magic_setregexp||| +magic_setsig||| +magic_setsubstr||| +magic_settaint||| +magic_setutf8||| +magic_setuvar||| +magic_setvec||| +magic_set||| +magic_sizepack||| +magic_wipepack||| +make_matcher||| +make_trie_failtable||| +make_trie||| +malloc_good_size|||n +malloced_size|||n +malloc||5.007002|n +markstack_grow||| +matcher_matches_sv||| +measure_struct||| +memEQ|5.004000||p +memNE|5.004000||p +mem_collxfrm||| +mem_log_common|||n +mess_alloc||| +mess_nocontext|||vn +mess||5.006000|v +method_common||| +mfree||5.007002|n +mg_clear||| +mg_copy||| +mg_dup||| +mg_find||| +mg_free||| +mg_get||| +mg_length||5.005000| +mg_localize||| +mg_magical||| +mg_set||| +mg_size||5.005000| +mini_mktime||5.007002| +missingterm||| +mode_from_discipline||| +modkids||| +mod||| +more_bodies||| +more_sv||| +moreswitches||| +mro_get_from_name||5.011000| +mro_get_linear_isa_dfs||| +mro_get_linear_isa||5.009005| +mro_get_private_data||5.011000| +mro_isa_changed_in||| +mro_meta_dup||| +mro_meta_init||| +mro_method_changed_in||5.009005| +mro_register||5.011000| +mro_set_mro||5.011000| +mro_set_private_data||5.011000| +mul128||| +mulexp10|||n +my_atof2||5.007002| +my_atof||5.006000| +my_attrs||| +my_bcopy|||n +my_betoh16|||n +my_betoh32|||n +my_betoh64|||n +my_betohi|||n +my_betohl|||n +my_betohs|||n +my_bzero|||n +my_chsize||| +my_clearenv||| +my_cxt_index||| +my_cxt_init||| +my_dirfd||5.009005| +my_exit_jump||| +my_exit||| +my_failure_exit||5.004000| +my_fflush_all||5.006000| +my_fork||5.007003|n +my_htobe16|||n +my_htobe32|||n +my_htobe64|||n +my_htobei|||n +my_htobel|||n +my_htobes|||n +my_htole16|||n +my_htole32|||n +my_htole64|||n +my_htolei|||n +my_htolel|||n +my_htoles|||n +my_htonl||| +my_kid||| +my_letoh16|||n +my_letoh32|||n +my_letoh64|||n +my_letohi|||n +my_letohl|||n +my_letohs|||n +my_lstat||| +my_memcmp||5.004000|n +my_memset|||n +my_ntohl||| +my_pclose||5.004000| +my_popen_list||5.007001| +my_popen||5.004000| +my_setenv||| +my_snprintf|5.009004||pvn +my_socketpair||5.007003|n +my_sprintf|5.009003||pvn +my_stat||| +my_strftime||5.007002| +my_strlcat|5.009004||pn +my_strlcpy|5.009004||pn +my_swabn|||n +my_swap||| +my_unexec||| +my_vsnprintf||5.009004|n +need_utf8|||n +newANONATTRSUB||5.006000| +newANONHASH||| +newANONLIST||| +newANONSUB||| +newASSIGNOP||| +newATTRSUB||5.006000| +newAVREF||| +newAV||| +newBINOP||| +newCONDOP||| +newCONSTSUB|5.004050||p +newCVREF||| +newDEFSVOP||| +newFORM||| +newFOROP||| +newGIVENOP||5.009003| +newGIVWHENOP||| +newGP||| +newGVOP||| +newGVREF||| +newGVgen||| +newHVREF||| +newHVhv||5.005000| +newHV||| +newIO||| +newLISTOP||| +newLOGOP||| +newLOOPEX||| +newLOOPOP||| +newMADPROP||| +newMADsv||| +newMYSUB||| +newNULLLIST||| +newOP||| +newPADOP||| +newPMOP||| +newPROG||| +newPVOP||| +newRANGE||| +newRV_inc|5.004000||p +newRV_noinc|5.004000||p +newRV||| +newSLICEOP||| +newSTATEOP||| +newSUB||| +newSVOP||| +newSVREF||| +newSV_type||5.009005| +newSVhek||5.009003| +newSViv||| +newSVnv||| +newSVpvf_nocontext|||vn +newSVpvf||5.004000|v +newSVpvn_flags|5.011000||p +newSVpvn_share|5.007001||p +newSVpvn_utf8|5.011000||p +newSVpvn|5.004050||p +newSVpvs_flags|5.011000||p +newSVpvs_share||5.009003| +newSVpvs|5.009003||p +newSVpv||| +newSVrv||| +newSVsv||| +newSVuv|5.006000||p +newSV||| +newTOKEN||| +newUNOP||| +newWHENOP||5.009003| +newWHILEOP||5.009003| +newXS_flags||5.009004| +newXSproto||5.006000| +newXS||5.006000| +new_collate||5.006000| +new_constant||| +new_ctype||5.006000| +new_he||| +new_logop||| +new_numeric||5.006000| +new_stackinfo||5.005000| +new_version||5.009000| +new_warnings_bitfield||| +next_symbol||| +nextargv||| +nextchar||| +ninstr||| +no_bareword_allowed||| +no_fh_allowed||| +no_op||| +not_a_number||| +nothreadhook||5.008000| +nuke_stacks||| +num_overflow|||n +offer_nice_chunk||| +oopsAV||| +oopsHV||| +op_clear||| +op_const_sv||| +op_dump||5.006000| +op_free||| +op_getmad_weak||| +op_getmad||| +op_null||5.007002| +op_refcnt_dec||| +op_refcnt_inc||| +op_refcnt_lock||5.009002| +op_refcnt_unlock||5.009002| +op_xmldump||| +open_script||| +pMY_CXT_|5.007003||p +pMY_CXT|5.007003||p +pTHX_|5.006000||p +pTHX|5.006000||p +packWARN|5.007003||p +pack_cat||5.007003| +pack_rec||| +package||| +packlist||5.008001| +pad_add_anon||| +pad_add_name||| +pad_alloc||| +pad_block_start||| +pad_check_dup||| +pad_compname_type||| +pad_findlex||| +pad_findmy||| +pad_fixup_inner_anons||| +pad_free||| +pad_leavemy||| +pad_new||| +pad_peg|||n +pad_push||| +pad_reset||| +pad_setsv||| +pad_sv||5.011000| +pad_swipe||| +pad_tidy||| +pad_undef||| +parse_body||| +parse_unicode_opts||| +parser_dup||| +parser_free||| +path_is_absolute|||n +peep||| +pending_Slabs_to_ro||| +perl_alloc_using|||n +perl_alloc|||n +perl_clone_using|||n +perl_clone|||n +perl_construct|||n +perl_destruct||5.007003|n +perl_free|||n +perl_parse||5.006000|n +perl_run|||n +pidgone||| +pm_description||| +pmflag||| +pmop_dump||5.006000| +pmop_xmldump||| +pmruntime||| +pmtrans||| +pop_scope||| +pregcomp||5.009005| +pregexec||| +pregfree2||5.011000| +pregfree||| +prepend_elem||| +prepend_madprops||| +printbuf||| +printf_nocontext|||vn +process_special_blocks||| +ptr_table_clear||5.009005| +ptr_table_fetch||5.009005| +ptr_table_find|||n +ptr_table_free||5.009005| +ptr_table_new||5.009005| +ptr_table_split||5.009005| +ptr_table_store||5.009005| +push_scope||| +put_byte||| +pv_display|5.006000||p +pv_escape|5.009004||p +pv_pretty|5.009004||p +pv_uni_display||5.007003| +qerror||| +qsortsvu||| +re_compile||5.009005| +re_croak2||| +re_dup_guts||| +re_intuit_start||5.009005| +re_intuit_string||5.006000| +readpipe_override||| +realloc||5.007002|n +reentrant_free||| +reentrant_init||| +reentrant_retry|||vn +reentrant_size||| +ref_array_or_hash||| +refcounted_he_chain_2hv||| +refcounted_he_fetch||| +refcounted_he_free||| +refcounted_he_new_common||| +refcounted_he_new||| +refcounted_he_value||| +refkids||| +refto||| +ref||5.011000| +reg_check_named_buff_matched||| +reg_named_buff_all||5.009005| +reg_named_buff_exists||5.009005| +reg_named_buff_fetch||5.009005| +reg_named_buff_firstkey||5.009005| +reg_named_buff_iter||| +reg_named_buff_nextkey||5.009005| +reg_named_buff_scalar||5.009005| +reg_named_buff||| +reg_namedseq||| +reg_node||| +reg_numbered_buff_fetch||| +reg_numbered_buff_length||| +reg_numbered_buff_store||| +reg_qr_package||| +reg_recode||| +reg_scan_name||| +reg_skipcomment||| +reg_temp_copy||| +reganode||| +regatom||| +regbranch||| +regclass_swash||5.009004| +regclass||| +regcppop||| +regcppush||| +regcurly|||n +regdump_extflags||| +regdump||5.005000| +regdupe_internal||| +regexec_flags||5.005000| +regfree_internal||5.009005| +reghop3|||n +reghop4|||n +reghopmaybe3|||n +reginclass||| +reginitcolors||5.006000| +reginsert||| +regmatch||| +regnext||5.005000| +regpiece||| +regpposixcc||| +regprop||| +regrepeat||| +regtail_study||| +regtail||| +regtry||| +reguni||| +regwhite|||n +reg||| +repeatcpy||| +report_evil_fh||| +report_uninit||| +require_pv||5.006000| +require_tie_mod||| +restore_magic||| +rninstr||| +rsignal_restore||| +rsignal_save||| +rsignal_state||5.004000| +rsignal||5.004000| +run_body||| +run_user_filter||| +runops_debug||5.005000| +runops_standard||5.005000| +rvpv_dup||| +rxres_free||| +rxres_restore||| +rxres_save||| +safesyscalloc||5.006000|n +safesysfree||5.006000|n +safesysmalloc||5.006000|n +safesysrealloc||5.006000|n +same_dirent||| +save_I16||5.004000| +save_I32||| +save_I8||5.006000| +save_adelete||5.011000| +save_aelem||5.004050| +save_alloc||5.006000| +save_aptr||| +save_ary||| +save_bool||5.008001| +save_clearsv||| +save_delete||| +save_destructor_x||5.006000| +save_destructor||5.006000| +save_freeop||| +save_freepv||| +save_freesv||| +save_generic_pvref||5.006001| +save_generic_svref||5.005030| +save_gp||5.004000| +save_hash||| +save_hek_flags|||n +save_helem_flags||5.011000| +save_helem||5.004050| +save_hints||| +save_hptr||| +save_int||| +save_item||| +save_iv||5.005000| +save_lines||| +save_list||| +save_long||| +save_magic||| +save_mortalizesv||5.007001| +save_nogv||| +save_op||| +save_padsv_and_mortalize||5.011000| +save_pptr||| +save_pushi32ptr||| +save_pushptri32ptr||| +save_pushptrptr||| +save_pushptr||5.011000| +save_re_context||5.006000| +save_scalar_at||| +save_scalar||| +save_set_svflags||5.009000| +save_shared_pvref||5.007003| +save_sptr||| +save_svref||| +save_vptr||5.006000| +savepvn||| +savepvs||5.009003| +savepv||| +savesharedpvn||5.009005| +savesharedpv||5.007003| +savestack_grow_cnt||5.008001| +savestack_grow||| +savesvpv||5.009002| +sawparens||| +scalar_mod_type|||n +scalarboolean||| +scalarkids||| +scalarseq||| +scalarvoid||| +scalar||| +scan_bin||5.006000| +scan_commit||| +scan_const||| +scan_formline||| +scan_heredoc||| +scan_hex||| +scan_ident||| +scan_inputsymbol||| +scan_num||5.007001| +scan_oct||| +scan_pat||| +scan_str||| +scan_subst||| +scan_trans||| +scan_version||5.009001| +scan_vstring||5.009005| +scan_word||| +scope||| +screaminstr||5.005000| +search_const||| +seed||5.008001| +sequence_num||| +sequence_tail||| +sequence||| +set_context||5.006000|n +set_numeric_local||5.006000| +set_numeric_radix||5.006000| +set_numeric_standard||5.006000| +setdefout||| +share_hek_flags||| +share_hek||5.004000| +si_dup||| +sighandler|||n +simplify_sort||| +skipspace0||| +skipspace1||| +skipspace2||| +skipspace||| +softref2xv||| +sortcv_stacked||| +sortcv_xsub||| +sortcv||| +sortsv_flags||5.009003| +sortsv||5.007003| +space_join_names_mortal||| +ss_dup||| +stack_grow||| +start_force||| +start_glob||| +start_subparse||5.004000| +stashpv_hvname_match||5.011000| +stdize_locale||| +store_cop_label||| +strEQ||| +strGE||| +strGT||| +strLE||| +strLT||| +strNE||| +str_to_version||5.006000| +strip_return||| +strnEQ||| +strnNE||| +study_chunk||| +sub_crush_depth||| +sublex_done||| +sublex_push||| +sublex_start||| +sv_2bool||| +sv_2cv||| +sv_2io||| +sv_2iuv_common||| +sv_2iuv_non_preserve||| +sv_2iv_flags||5.009001| +sv_2iv||| +sv_2mortal||| +sv_2num||| +sv_2nv||| +sv_2pv_flags|5.007002||p +sv_2pv_nolen|5.006000||p +sv_2pvbyte_nolen|5.006000||p +sv_2pvbyte|5.006000||p +sv_2pvutf8_nolen||5.006000| +sv_2pvutf8||5.006000| +sv_2pv||| +sv_2uv_flags||5.009001| +sv_2uv|5.004000||p +sv_add_arena||| +sv_add_backref||| +sv_backoff||| +sv_bless||| +sv_cat_decode||5.008001| +sv_catpv_mg|5.004050||p +sv_catpvf_mg_nocontext|||pvn +sv_catpvf_mg|5.006000|5.004000|pv +sv_catpvf_nocontext|||vn +sv_catpvf||5.004000|v +sv_catpvn_flags||5.007002| +sv_catpvn_mg|5.004050||p +sv_catpvn_nomg|5.007002||p +sv_catpvn||| +sv_catpvs|5.009003||p +sv_catpv||| +sv_catsv_flags||5.007002| +sv_catsv_mg|5.004050||p +sv_catsv_nomg|5.007002||p +sv_catsv||| +sv_catxmlpvn||| +sv_catxmlsv||| +sv_chop||| +sv_clean_all||| +sv_clean_objs||| +sv_clear||| +sv_cmp_locale||5.004000| +sv_cmp||| +sv_collxfrm||| +sv_compile_2op||5.008001| +sv_copypv||5.007003| +sv_dec||| +sv_del_backref||| +sv_derived_from||5.004000| +sv_destroyable||5.010000| +sv_does||5.009004| +sv_dump||| +sv_dup||| +sv_eq||| +sv_exp_grow||| +sv_force_normal_flags||5.007001| +sv_force_normal||5.006000| +sv_free2||| +sv_free_arenas||| +sv_free||| +sv_gets||5.004000| +sv_grow||| +sv_i_ncmp||| +sv_inc||| +sv_insert_flags||5.011000| +sv_insert||| +sv_isa||| +sv_isobject||| +sv_iv||5.005000| +sv_kill_backrefs||| +sv_len_utf8||5.006000| +sv_len||| +sv_magic_portable|5.011000|5.004000|p +sv_magicext||5.007003| +sv_magic||| +sv_mortalcopy||| +sv_ncmp||| +sv_newmortal||| +sv_newref||| +sv_nolocking||5.007003| +sv_nosharing||5.007003| +sv_nounlocking||| +sv_nv||5.005000| +sv_peek||5.005000| +sv_pos_b2u_midway||| +sv_pos_b2u||5.006000| +sv_pos_u2b_cached||| +sv_pos_u2b_forwards|||n +sv_pos_u2b_midway|||n +sv_pos_u2b||5.006000| +sv_pvbyten_force||5.006000| +sv_pvbyten||5.006000| +sv_pvbyte||5.006000| +sv_pvn_force_flags|5.007002||p +sv_pvn_force||| +sv_pvn_nomg|5.007003|5.005000|p +sv_pvn||5.005000| +sv_pvutf8n_force||5.006000| +sv_pvutf8n||5.006000| +sv_pvutf8||5.006000| +sv_pv||5.006000| +sv_recode_to_utf8||5.007003| +sv_reftype||| +sv_release_COW||| +sv_replace||| +sv_report_used||| +sv_reset||| +sv_rvweaken||5.006000| +sv_setiv_mg|5.004050||p +sv_setiv||| +sv_setnv_mg|5.006000||p +sv_setnv||| +sv_setpv_mg|5.004050||p +sv_setpvf_mg_nocontext|||pvn +sv_setpvf_mg|5.006000|5.004000|pv +sv_setpvf_nocontext|||vn +sv_setpvf||5.004000|v +sv_setpviv_mg||5.008001| +sv_setpviv||5.008001| +sv_setpvn_mg|5.004050||p +sv_setpvn||| +sv_setpvs|5.009004||p +sv_setpv||| +sv_setref_iv||| +sv_setref_nv||| +sv_setref_pvn||| +sv_setref_pv||| +sv_setref_uv||5.007001| +sv_setsv_cow||| +sv_setsv_flags||5.007002| +sv_setsv_mg|5.004050||p +sv_setsv_nomg|5.007002||p +sv_setsv||| +sv_setuv_mg|5.004050||p +sv_setuv|5.004000||p +sv_tainted||5.004000| +sv_taint||5.004000| +sv_true||5.005000| +sv_unglob||| +sv_uni_display||5.007003| +sv_unmagic||| +sv_unref_flags||5.007001| +sv_unref||| +sv_untaint||5.004000| +sv_upgrade||| +sv_usepvn_flags||5.009004| +sv_usepvn_mg|5.004050||p +sv_usepvn||| +sv_utf8_decode||5.006000| +sv_utf8_downgrade||5.006000| +sv_utf8_encode||5.006000| +sv_utf8_upgrade_flags_grow||5.011000| +sv_utf8_upgrade_flags||5.007002| +sv_utf8_upgrade_nomg||5.007002| +sv_utf8_upgrade||5.007001| +sv_uv|5.005000||p +sv_vcatpvf_mg|5.006000|5.004000|p +sv_vcatpvfn||5.004000| +sv_vcatpvf|5.006000|5.004000|p +sv_vsetpvf_mg|5.006000|5.004000|p +sv_vsetpvfn||5.004000| +sv_vsetpvf|5.006000|5.004000|p +sv_xmlpeek||| +svtype||| +swallow_bom||| +swap_match_buff||| +swash_fetch||5.007002| +swash_get||| +swash_init||5.006000| +sys_init3||5.010000|n +sys_init||5.010000|n +sys_intern_clear||| +sys_intern_dup||| +sys_intern_init||| +sys_term||5.010000|n +taint_env||| +taint_proper||| +tmps_grow||5.006000| +toLOWER||| +toUPPER||| +to_byte_substr||| +to_uni_fold||5.007003| +to_uni_lower_lc||5.006000| +to_uni_lower||5.007003| +to_uni_title_lc||5.006000| +to_uni_title||5.007003| +to_uni_upper_lc||5.006000| +to_uni_upper||5.007003| +to_utf8_case||5.007003| +to_utf8_fold||5.007003| +to_utf8_lower||5.007003| +to_utf8_substr||| +to_utf8_title||5.007003| +to_utf8_upper||5.007003| +token_free||| +token_getmad||| +tokenize_use||| +tokeq||| +tokereport||| +too_few_arguments||| +too_many_arguments||| +uiv_2buf|||n +unlnk||| +unpack_rec||| +unpack_str||5.007003| +unpackstring||5.008001| +unshare_hek_or_pvn||| +unshare_hek||| +unsharepvn||5.004000| +unwind_handler_stack||| +update_debugger_info||| +upg_version||5.009005| +usage||| +utf16_to_utf8_reversed||5.006001| +utf16_to_utf8||5.006001| +utf8_distance||5.006000| +utf8_hop||5.006000| +utf8_length||5.007001| +utf8_mg_pos_cache_update||| +utf8_to_bytes||5.006001| +utf8_to_uvchr||5.007001| +utf8_to_uvuni||5.007001| +utf8n_to_uvchr||| +utf8n_to_uvuni||5.007001| +utilize||| +uvchr_to_utf8_flags||5.007003| +uvchr_to_utf8||| +uvuni_to_utf8_flags||5.007003| +uvuni_to_utf8||5.007001| +validate_suid||| +varname||| +vcmp||5.009000| +vcroak||5.006000| +vdeb||5.007003| +vdie_common||| +vdie_croak_common||| +vdie||| +vform||5.006000| +visit||| +vivify_defelem||| +vivify_ref||| +vload_module|5.006000||p +vmess||5.006000| +vnewSVpvf|5.006000|5.004000|p +vnormal||5.009002| +vnumify||5.009000| +vstringify||5.009000| +vverify||5.009003| +vwarner||5.006000| +vwarn||5.006000| +wait4pid||| +warn_nocontext|||vn +warner_nocontext|||vn +warner|5.006000|5.004000|pv +warn|||v +watch||| +whichsig||| +write_no_mem||| +write_to_stderr||| +xmldump_all||| +xmldump_attr||| +xmldump_eval||| +xmldump_form||| +xmldump_indent|||v +xmldump_packsubs||| +xmldump_sub||| +xmldump_vindent||| +yyerror||| +yylex||| +yyparse||| +yywarn||| +); + +if (exists $opt{'list-unsupported'}) { + my $f; + for $f (sort { lc $a cmp lc $b } keys %API) { + next unless $API{$f}{todo}; + print "$f ", '.'x(40-length($f)), " ", format_version($API{$f}{todo}), "\n"; + } + exit 0; +} + +# Scan for possible replacement candidates + +my(%replace, %need, %hints, %warnings, %depends); +my $replace = 0; +my($hint, $define, $function); + +sub find_api +{ + my $code = shift; + $code =~ s{ + / (?: \*[^*]*\*+(?:[^$ccs][^*]*\*+)* / | /[^\r\n]*) + | "[^"\\]*(?:\\.[^"\\]*)*" + | '[^'\\]*(?:\\.[^'\\]*)*' }{}egsx; + grep { exists $API{$_} } $code =~ /(\w+)/mg; +} + +while () { + if ($hint) { + my $h = $hint->[0] eq 'Hint' ? \%hints : \%warnings; + if (m{^\s*\*\s(.*?)\s*$}) { + for (@{$hint->[1]}) { + $h->{$_} ||= ''; # suppress warning with older perls + $h->{$_} .= "$1\n"; + } + } + else { undef $hint } + } + + $hint = [$1, [split /,?\s+/, $2]] + if m{^\s*$rccs\s+(Hint|Warning):\s+(\w+(?:,?\s+\w+)*)\s*$}; + + if ($define) { + if ($define->[1] =~ /\\$/) { + $define->[1] .= $_; + } + else { + if (exists $API{$define->[0]} && $define->[1] !~ /^DPPP_\(/) { + my @n = find_api($define->[1]); + push @{$depends{$define->[0]}}, @n if @n + } + undef $define; + } + } + + $define = [$1, $2] if m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(.*)}; + + if ($function) { + if (/^}/) { + if (exists $API{$function->[0]}) { + my @n = find_api($function->[1]); + push @{$depends{$function->[0]}}, @n if @n + } + undef $function; + } + else { + $function->[1] .= $_; + } + } + + $function = [$1, ''] if m{^DPPP_\(my_(\w+)\)}; + + $replace = $1 if m{^\s*$rccs\s+Replace:\s+(\d+)\s+$rcce\s*$}; + $replace{$2} = $1 if $replace and m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(\w+)}; + $replace{$2} = $1 if m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(\w+).*$rccs\s+Replace\s+$rcce}; + $replace{$1} = $2 if m{^\s*$rccs\s+Replace (\w+) with (\w+)\s+$rcce\s*$}; + + if (m{^\s*$rccs\s+(\w+(\s*,\s*\w+)*)\s+depends\s+on\s+(\w+(\s*,\s*\w+)*)\s+$rcce\s*$}) { + my @deps = map { s/\s+//g; $_ } split /,/, $3; + my $d; + for $d (map { s/\s+//g; $_ } split /,/, $1) { + push @{$depends{$d}}, @deps; + } + } + + $need{$1} = 1 if m{^#if\s+defined\(NEED_(\w+)(?:_GLOBAL)?\)}; +} + +for (values %depends) { + my %s; + $_ = [sort grep !$s{$_}++, @$_]; +} + +if (exists $opt{'api-info'}) { + my $f; + my $count = 0; + my $match = $opt{'api-info'} =~ m!^/(.*)/$! ? $1 : "^\Q$opt{'api-info'}\E\$"; + for $f (sort { lc $a cmp lc $b } keys %API) { + next unless $f =~ /$match/; + print "\n=== $f ===\n\n"; + my $info = 0; + if ($API{$f}{base} || $API{$f}{todo}) { + my $base = format_version($API{$f}{base} || $API{$f}{todo}); + print "Supported at least starting from perl-$base.\n"; + $info++; + } + if ($API{$f}{provided}) { + my $todo = $API{$f}{todo} ? format_version($API{$f}{todo}) : "5.003"; + print "Support by $ppport provided back to perl-$todo.\n"; + print "Support needs to be explicitly requested by NEED_$f.\n" if exists $need{$f}; + print "Depends on: ", join(', ', @{$depends{$f}}), ".\n" if exists $depends{$f}; + print "\n$hints{$f}" if exists $hints{$f}; + print "\nWARNING:\n$warnings{$f}" if exists $warnings{$f}; + $info++; + } + print "No portability information available.\n" unless $info; + $count++; + } + $count or print "Found no API matching '$opt{'api-info'}'."; + print "\n"; + exit 0; +} + +if (exists $opt{'list-provided'}) { + my $f; + for $f (sort { lc $a cmp lc $b } keys %API) { + next unless $API{$f}{provided}; + my @flags; + push @flags, 'explicit' if exists $need{$f}; + push @flags, 'depend' if exists $depends{$f}; + push @flags, 'hint' if exists $hints{$f}; + push @flags, 'warning' if exists $warnings{$f}; + my $flags = @flags ? ' ['.join(', ', @flags).']' : ''; + print "$f$flags\n"; + } + exit 0; +} + +my @files; +my @srcext = qw( .xs .c .h .cc .cpp -c.inc -xs.inc ); +my $srcext = join '|', map { quotemeta $_ } @srcext; + +if (@ARGV) { + my %seen; + for (@ARGV) { + if (-e) { + if (-f) { + push @files, $_ unless $seen{$_}++; + } + else { warn "'$_' is not a file.\n" } + } + else { + my @new = grep { -f } glob $_ + or warn "'$_' does not exist.\n"; + push @files, grep { !$seen{$_}++ } @new; + } + } +} +else { + eval { + require File::Find; + File::Find::find(sub { + $File::Find::name =~ /($srcext)$/i + and push @files, $File::Find::name; + }, '.'); + }; + if ($@) { + @files = map { glob "*$_" } @srcext; + } +} + +if (!@ARGV || $opt{filter}) { + my(@in, @out); + my %xsc = map { /(.*)\.xs$/ ? ("$1.c" => 1, "$1.cc" => 1) : () } @files; + for (@files) { + my $out = exists $xsc{$_} || /\b\Q$ppport\E$/i || !/($srcext)$/i; + push @{ $out ? \@out : \@in }, $_; + } + if (@ARGV && @out) { + warning("Skipping the following files (use --nofilter to avoid this):\n| ", join "\n| ", @out); + } + @files = @in; +} + +die "No input files given!\n" unless @files; + +my(%files, %global, %revreplace); +%revreplace = reverse %replace; +my $filename; +my $patch_opened = 0; + +for $filename (@files) { + unless (open IN, "<$filename") { + warn "Unable to read from $filename: $!\n"; + next; + } + + info("Scanning $filename ..."); + + my $c = do { local $/; }; + close IN; + + my %file = (orig => $c, changes => 0); + + # Temporarily remove C/XS comments and strings from the code + my @ccom; + + $c =~ s{ + ( ^$HS*\#$HS*include\b[^\r\n]+\b(?:\Q$ppport\E|XSUB\.h)\b[^\r\n]* + | ^$HS*\#$HS*(?:define|elif|if(?:def)?)\b[^\r\n]* ) + | ( ^$HS*\#[^\r\n]* + | "[^"\\]*(?:\\.[^"\\]*)*" + | '[^'\\]*(?:\\.[^'\\]*)*' + | / (?: \*[^*]*\*+(?:[^$ccs][^*]*\*+)* / | /[^\r\n]* ) ) + }{ defined $2 and push @ccom, $2; + defined $1 ? $1 : "$ccs$#ccom$cce" }mgsex; + + $file{ccom} = \@ccom; + $file{code} = $c; + $file{has_inc_ppport} = $c =~ /^$HS*#$HS*include[^\r\n]+\b\Q$ppport\E\b/m; + + my $func; + + for $func (keys %API) { + my $match = $func; + $match .= "|$revreplace{$func}" if exists $revreplace{$func}; + if ($c =~ /\b(?:Perl_)?($match)\b/) { + $file{uses_replace}{$1}++ if exists $revreplace{$func} && $1 eq $revreplace{$func}; + $file{uses_Perl}{$func}++ if $c =~ /\bPerl_$func\b/; + if (exists $API{$func}{provided}) { + $file{uses_provided}{$func}++; + if (!exists $API{$func}{base} || $API{$func}{base} > $opt{'compat-version'}) { + $file{uses}{$func}++; + my @deps = rec_depend($func); + if (@deps) { + $file{uses_deps}{$func} = \@deps; + for (@deps) { + $file{uses}{$_} = 0 unless exists $file{uses}{$_}; + } + } + for ($func, @deps) { + $file{needs}{$_} = 'static' if exists $need{$_}; + } + } + } + if (exists $API{$func}{todo} && $API{$func}{todo} > $opt{'compat-version'}) { + if ($c =~ /\b$func\b/) { + $file{uses_todo}{$func}++; + } + } + } + } + + while ($c =~ /^$HS*#$HS*define$HS+(NEED_(\w+?)(_GLOBAL)?)\b/mg) { + if (exists $need{$2}) { + $file{defined $3 ? 'needed_global' : 'needed_static'}{$2}++; + } + else { warning("Possibly wrong #define $1 in $filename") } + } + + for (qw(uses needs uses_todo needed_global needed_static)) { + for $func (keys %{$file{$_}}) { + push @{$global{$_}{$func}}, $filename; + } + } + + $files{$filename} = \%file; +} + +# Globally resolve NEED_'s +my $need; +for $need (keys %{$global{needs}}) { + if (@{$global{needs}{$need}} > 1) { + my @targets = @{$global{needs}{$need}}; + my @t = grep $files{$_}{needed_global}{$need}, @targets; + @targets = @t if @t; + @t = grep /\.xs$/i, @targets; + @targets = @t if @t; + my $target = shift @targets; + $files{$target}{needs}{$need} = 'global'; + for (@{$global{needs}{$need}}) { + $files{$_}{needs}{$need} = 'extern' if $_ ne $target; + } + } +} + +for $filename (@files) { + exists $files{$filename} or next; + + info("=== Analyzing $filename ==="); + + my %file = %{$files{$filename}}; + my $func; + my $c = $file{code}; + my $warnings = 0; + + for $func (sort keys %{$file{uses_Perl}}) { + if ($API{$func}{varargs}) { + unless ($API{$func}{nothxarg}) { + my $changes = ($c =~ s{\b(Perl_$func\s*\(\s*)(?!aTHX_?)(\)|[^\s)]*\))} + { $1 . ($2 eq ')' ? 'aTHX' : 'aTHX_ ') . $2 }ge); + if ($changes) { + warning("Doesn't pass interpreter argument aTHX to Perl_$func"); + $file{changes} += $changes; + } + } + } + else { + warning("Uses Perl_$func instead of $func"); + $file{changes} += ($c =~ s{\bPerl_$func(\s*)\((\s*aTHX_?)?\s*} + {$func$1(}g); + } + } + + for $func (sort keys %{$file{uses_replace}}) { + warning("Uses $func instead of $replace{$func}"); + $file{changes} += ($c =~ s/\b$func\b/$replace{$func}/g); + } + + for $func (sort keys %{$file{uses_provided}}) { + if ($file{uses}{$func}) { + if (exists $file{uses_deps}{$func}) { + diag("Uses $func, which depends on ", join(', ', @{$file{uses_deps}{$func}})); + } + else { + diag("Uses $func"); + } + } + $warnings += hint($func); + } + + unless ($opt{quiet}) { + for $func (sort keys %{$file{uses_todo}}) { + print "*** WARNING: Uses $func, which may not be portable below perl ", + format_version($API{$func}{todo}), ", even with '$ppport'\n"; + $warnings++; + } + } + + for $func (sort keys %{$file{needed_static}}) { + my $message = ''; + if (not exists $file{uses}{$func}) { + $message = "No need to define NEED_$func if $func is never used"; + } + elsif (exists $file{needs}{$func} && $file{needs}{$func} ne 'static') { + $message = "No need to define NEED_$func when already needed globally"; + } + if ($message) { + diag($message); + $file{changes} += ($c =~ s/^$HS*#$HS*define$HS+NEED_$func\b.*$LF//mg); + } + } + + for $func (sort keys %{$file{needed_global}}) { + my $message = ''; + if (not exists $global{uses}{$func}) { + $message = "No need to define NEED_${func}_GLOBAL if $func is never used"; + } + elsif (exists $file{needs}{$func}) { + if ($file{needs}{$func} eq 'extern') { + $message = "No need to define NEED_${func}_GLOBAL when already needed globally"; + } + elsif ($file{needs}{$func} eq 'static') { + $message = "No need to define NEED_${func}_GLOBAL when only used in this file"; + } + } + if ($message) { + diag($message); + $file{changes} += ($c =~ s/^$HS*#$HS*define$HS+NEED_${func}_GLOBAL\b.*$LF//mg); + } + } + + $file{needs_inc_ppport} = keys %{$file{uses}}; + + if ($file{needs_inc_ppport}) { + my $pp = ''; + + for $func (sort keys %{$file{needs}}) { + my $type = $file{needs}{$func}; + next if $type eq 'extern'; + my $suffix = $type eq 'global' ? '_GLOBAL' : ''; + unless (exists $file{"needed_$type"}{$func}) { + if ($type eq 'global') { + diag("Files [@{$global{needs}{$func}}] need $func, adding global request"); + } + else { + diag("File needs $func, adding static request"); + } + $pp .= "#define NEED_$func$suffix\n"; + } + } + + if ($pp && ($c =~ s/^(?=$HS*#$HS*define$HS+NEED_\w+)/$pp/m)) { + $pp = ''; + $file{changes}++; + } + + unless ($file{has_inc_ppport}) { + diag("Needs to include '$ppport'"); + $pp .= qq(#include "$ppport"\n) + } + + if ($pp) { + $file{changes} += ($c =~ s/^($HS*#$HS*define$HS+NEED_\w+.*?)^/$1$pp/ms) + || ($c =~ s/^(?=$HS*#$HS*include.*\Q$ppport\E)/$pp/m) + || ($c =~ s/^($HS*#$HS*include.*XSUB.*\s*?)^/$1$pp/m) + || ($c =~ s/^/$pp/); + } + } + else { + if ($file{has_inc_ppport}) { + diag("No need to include '$ppport'"); + $file{changes} += ($c =~ s/^$HS*?#$HS*include.*\Q$ppport\E.*?$LF//m); + } + } + + # put back in our C comments + my $ix; + my $cppc = 0; + my @ccom = @{$file{ccom}}; + for $ix (0 .. $#ccom) { + if (!$opt{cplusplus} && $ccom[$ix] =~ s!^//!!) { + $cppc++; + $file{changes} += $c =~ s/$rccs$ix$rcce/$ccs$ccom[$ix] $cce/; + } + else { + $c =~ s/$rccs$ix$rcce/$ccom[$ix]/; + } + } + + if ($cppc) { + my $s = $cppc != 1 ? 's' : ''; + warning("Uses $cppc C++ style comment$s, which is not portable"); + } + + my $s = $warnings != 1 ? 's' : ''; + my $warn = $warnings ? " ($warnings warning$s)" : ''; + info("Analysis completed$warn"); + + if ($file{changes}) { + if (exists $opt{copy}) { + my $newfile = "$filename$opt{copy}"; + if (-e $newfile) { + error("'$newfile' already exists, refusing to write copy of '$filename'"); + } + else { + local *F; + if (open F, ">$newfile") { + info("Writing copy of '$filename' with changes to '$newfile'"); + print F $c; + close F; + } + else { + error("Cannot open '$newfile' for writing: $!"); + } + } + } + elsif (exists $opt{patch} || $opt{changes}) { + if (exists $opt{patch}) { + unless ($patch_opened) { + if (open PATCH, ">$opt{patch}") { + $patch_opened = 1; + } + else { + error("Cannot open '$opt{patch}' for writing: $!"); + delete $opt{patch}; + $opt{changes} = 1; + goto fallback; + } + } + mydiff(\*PATCH, $filename, $c); + } + else { +fallback: + info("Suggested changes:"); + mydiff(\*STDOUT, $filename, $c); + } + } + else { + my $s = $file{changes} == 1 ? '' : 's'; + info("$file{changes} potentially required change$s detected"); + } + } + else { + info("Looks good"); + } +} + +close PATCH if $patch_opened; + +exit 0; + + +sub try_use { eval "use @_;"; return $@ eq '' } + +sub mydiff +{ + local *F = shift; + my($file, $str) = @_; + my $diff; + + if (exists $opt{diff}) { + $diff = run_diff($opt{diff}, $file, $str); + } + + if (!defined $diff and try_use('Text::Diff')) { + $diff = Text::Diff::diff($file, \$str, { STYLE => 'Unified' }); + $diff = <
$tmp") { + print F $str; + close F; + + if (open F, "$prog $file $tmp |") { + while () { + s/\Q$tmp\E/$file.patched/; + $diff .= $_; + } + close F; + unlink $tmp; + return $diff; + } + + unlink $tmp; + } + else { + error("Cannot open '$tmp' for writing: $!"); + } + + return undef; +} + +sub rec_depend +{ + my($func, $seen) = @_; + return () unless exists $depends{$func}; + $seen = {%{$seen||{}}}; + return () if $seen->{$func}++; + my %s; + grep !$s{$_}++, map { ($_, rec_depend($_, $seen)) } @{$depends{$func}}; +} + +sub parse_version +{ + my $ver = shift; + + if ($ver =~ /^(\d+)\.(\d+)\.(\d+)$/) { + return ($1, $2, $3); + } + elsif ($ver !~ /^\d+\.[\d_]+$/) { + die "cannot parse version '$ver'\n"; + } + + $ver =~ s/_//g; + $ver =~ s/$/000000/; + + my($r,$v,$s) = $ver =~ /(\d+)\.(\d{3})(\d{3})/; + + $v = int $v; + $s = int $s; + + if ($r < 5 || ($r == 5 && $v < 6)) { + if ($s % 10) { + die "cannot parse version '$ver'\n"; + } + } + + return ($r, $v, $s); +} + +sub format_version +{ + my $ver = shift; + + $ver =~ s/$/000000/; + my($r,$v,$s) = $ver =~ /(\d+)\.(\d{3})(\d{3})/; + + $v = int $v; + $s = int $s; + + if ($r < 5 || ($r == 5 && $v < 6)) { + if ($s % 10) { + die "invalid version '$ver'\n"; + } + $s /= 10; + + $ver = sprintf "%d.%03d", $r, $v; + $s > 0 and $ver .= sprintf "_%02d", $s; + + return $ver; + } + + return sprintf "%d.%d.%d", $r, $v, $s; +} + +sub info +{ + $opt{quiet} and return; + print @_, "\n"; +} + +sub diag +{ + $opt{quiet} and return; + $opt{diag} and print @_, "\n"; +} + +sub warning +{ + $opt{quiet} and return; + print "*** ", @_, "\n"; +} + +sub error +{ + print "*** ERROR: ", @_, "\n"; +} + +my %given_hints; +my %given_warnings; +sub hint +{ + $opt{quiet} and return; + my $func = shift; + my $rv = 0; + if (exists $warnings{$func} && !$given_warnings{$func}++) { + my $warn = $warnings{$func}; + $warn =~ s!^!*** !mg; + print "*** WARNING: $func\n", $warn; + $rv++; + } + if ($opt{hints} && exists $hints{$func} && !$given_hints{$func}++) { + my $hint = $hints{$func}; + $hint =~ s/^/ /mg; + print " --- hint for $func ---\n", $hint; + } + $rv; +} + +sub usage +{ + my($usage) = do { local(@ARGV,$/)=($0); <> } =~ /^=head\d$HS+SYNOPSIS\s*^(.*?)\s*^=/ms; + my %M = ( 'I' => '*' ); + $usage =~ s/^\s*perl\s+\S+/$^X $0/; + $usage =~ s/([A-Z])<([^>]+)>/$M{$1}$2$M{$1}/g; + + print < }; + my($copy) = $self =~ /^=head\d\s+COPYRIGHT\s*^(.*?)^=\w+/ms; + $copy =~ s/^(?=\S+)/ /gms; + $self =~ s/^$HS+Do NOT edit.*?(?=^-)/$copy/ms; + $self =~ s/^SKIP.*(?=^__DATA__)/SKIP +if (\@ARGV && \$ARGV[0] eq '--unstrip') { + eval { require Devel::PPPort }; + \$@ and die "Cannot require Devel::PPPort, please install.\\n"; + if (eval \$Devel::PPPort::VERSION < $VERSION) { + die "$0 was originally generated with Devel::PPPort $VERSION.\\n" + . "Your Devel::PPPort is only version \$Devel::PPPort::VERSION.\\n" + . "Please install a newer version, or --unstrip will not work.\\n"; + } + Devel::PPPort::WriteFile(\$0); + exit 0; +} +print <$0" or die "cannot strip $0: $!\n"; + print OUT "$pl$c\n"; + + exit 0; +} + +__DATA__ +*/ + +#ifndef _P_P_PORTABILITY_H_ +#define _P_P_PORTABILITY_H_ + +#ifndef DPPP_NAMESPACE +# define DPPP_NAMESPACE DPPP_ +#endif + +#define DPPP_CAT2(x,y) CAT2(x,y) +#define DPPP_(name) DPPP_CAT2(DPPP_NAMESPACE, name) + +#ifndef PERL_REVISION +# if !defined(__PATCHLEVEL_H_INCLUDED__) && !(defined(PATCHLEVEL) && defined(SUBVERSION)) +# define PERL_PATCHLEVEL_H_IMPLICIT +# include +# endif +# if !(defined(PERL_VERSION) || (defined(SUBVERSION) && defined(PATCHLEVEL))) +# include +# endif +# ifndef PERL_REVISION +# define PERL_REVISION (5) + /* Replace: 1 */ +# define PERL_VERSION PATCHLEVEL +# define PERL_SUBVERSION SUBVERSION + /* Replace PERL_PATCHLEVEL with PERL_VERSION */ + /* Replace: 0 */ +# endif +#endif + +#define _dpppDEC2BCD(dec) ((((dec)/100)<<8)|((((dec)%100)/10)<<4)|((dec)%10)) +#define PERL_BCDVERSION ((_dpppDEC2BCD(PERL_REVISION)<<24)|(_dpppDEC2BCD(PERL_VERSION)<<12)|_dpppDEC2BCD(PERL_SUBVERSION)) + +/* It is very unlikely that anyone will try to use this with Perl 6 + (or greater), but who knows. + */ +#if PERL_REVISION != 5 +# error ppport.h only works with Perl version 5 +#endif /* PERL_REVISION != 5 */ +#ifndef dTHR +# define dTHR dNOOP +#endif +#ifndef dTHX +# define dTHX dNOOP +#endif + +#ifndef dTHXa +# define dTHXa(x) dNOOP +#endif +#ifndef pTHX +# define pTHX void +#endif + +#ifndef pTHX_ +# define pTHX_ +#endif + +#ifndef aTHX +# define aTHX +#endif + +#ifndef aTHX_ +# define aTHX_ +#endif + +#if (PERL_BCDVERSION < 0x5006000) +# ifdef USE_THREADS +# define aTHXR thr +# define aTHXR_ thr, +# else +# define aTHXR +# define aTHXR_ +# endif +# define dTHXR dTHR +#else +# define aTHXR aTHX +# define aTHXR_ aTHX_ +# define dTHXR dTHX +#endif +#ifndef dTHXoa +# define dTHXoa(x) dTHXa(x) +#endif + +#ifdef I_LIMITS +# include +#endif + +#ifndef PERL_UCHAR_MIN +# define PERL_UCHAR_MIN ((unsigned char)0) +#endif + +#ifndef PERL_UCHAR_MAX +# ifdef UCHAR_MAX +# define PERL_UCHAR_MAX ((unsigned char)UCHAR_MAX) +# else +# ifdef MAXUCHAR +# define PERL_UCHAR_MAX ((unsigned char)MAXUCHAR) +# else +# define PERL_UCHAR_MAX ((unsigned char)~(unsigned)0) +# endif +# endif +#endif + +#ifndef PERL_USHORT_MIN +# define PERL_USHORT_MIN ((unsigned short)0) +#endif + +#ifndef PERL_USHORT_MAX +# ifdef USHORT_MAX +# define PERL_USHORT_MAX ((unsigned short)USHORT_MAX) +# else +# ifdef MAXUSHORT +# define PERL_USHORT_MAX ((unsigned short)MAXUSHORT) +# else +# ifdef USHRT_MAX +# define PERL_USHORT_MAX ((unsigned short)USHRT_MAX) +# else +# define PERL_USHORT_MAX ((unsigned short)~(unsigned)0) +# endif +# endif +# endif +#endif + +#ifndef PERL_SHORT_MAX +# ifdef SHORT_MAX +# define PERL_SHORT_MAX ((short)SHORT_MAX) +# else +# ifdef MAXSHORT /* Often used in */ +# define PERL_SHORT_MAX ((short)MAXSHORT) +# else +# ifdef SHRT_MAX +# define PERL_SHORT_MAX ((short)SHRT_MAX) +# else +# define PERL_SHORT_MAX ((short) (PERL_USHORT_MAX >> 1)) +# endif +# endif +# endif +#endif + +#ifndef PERL_SHORT_MIN +# ifdef SHORT_MIN +# define PERL_SHORT_MIN ((short)SHORT_MIN) +# else +# ifdef MINSHORT +# define PERL_SHORT_MIN ((short)MINSHORT) +# else +# ifdef SHRT_MIN +# define PERL_SHORT_MIN ((short)SHRT_MIN) +# else +# define PERL_SHORT_MIN (-PERL_SHORT_MAX - ((3 & -1) == 3)) +# endif +# endif +# endif +#endif + +#ifndef PERL_UINT_MAX +# ifdef UINT_MAX +# define PERL_UINT_MAX ((unsigned int)UINT_MAX) +# else +# ifdef MAXUINT +# define PERL_UINT_MAX ((unsigned int)MAXUINT) +# else +# define PERL_UINT_MAX (~(unsigned int)0) +# endif +# endif +#endif + +#ifndef PERL_UINT_MIN +# define PERL_UINT_MIN ((unsigned int)0) +#endif + +#ifndef PERL_INT_MAX +# ifdef INT_MAX +# define PERL_INT_MAX ((int)INT_MAX) +# else +# ifdef MAXINT /* Often used in */ +# define PERL_INT_MAX ((int)MAXINT) +# else +# define PERL_INT_MAX ((int)(PERL_UINT_MAX >> 1)) +# endif +# endif +#endif + +#ifndef PERL_INT_MIN +# ifdef INT_MIN +# define PERL_INT_MIN ((int)INT_MIN) +# else +# ifdef MININT +# define PERL_INT_MIN ((int)MININT) +# else +# define PERL_INT_MIN (-PERL_INT_MAX - ((3 & -1) == 3)) +# endif +# endif +#endif + +#ifndef PERL_ULONG_MAX +# ifdef ULONG_MAX +# define PERL_ULONG_MAX ((unsigned long)ULONG_MAX) +# else +# ifdef MAXULONG +# define PERL_ULONG_MAX ((unsigned long)MAXULONG) +# else +# define PERL_ULONG_MAX (~(unsigned long)0) +# endif +# endif +#endif + +#ifndef PERL_ULONG_MIN +# define PERL_ULONG_MIN ((unsigned long)0L) +#endif + +#ifndef PERL_LONG_MAX +# ifdef LONG_MAX +# define PERL_LONG_MAX ((long)LONG_MAX) +# else +# ifdef MAXLONG +# define PERL_LONG_MAX ((long)MAXLONG) +# else +# define PERL_LONG_MAX ((long) (PERL_ULONG_MAX >> 1)) +# endif +# endif +#endif + +#ifndef PERL_LONG_MIN +# ifdef LONG_MIN +# define PERL_LONG_MIN ((long)LONG_MIN) +# else +# ifdef MINLONG +# define PERL_LONG_MIN ((long)MINLONG) +# else +# define PERL_LONG_MIN (-PERL_LONG_MAX - ((3 & -1) == 3)) +# endif +# endif +#endif + +#if defined(HAS_QUAD) && (defined(convex) || defined(uts)) +# ifndef PERL_UQUAD_MAX +# ifdef ULONGLONG_MAX +# define PERL_UQUAD_MAX ((unsigned long long)ULONGLONG_MAX) +# else +# ifdef MAXULONGLONG +# define PERL_UQUAD_MAX ((unsigned long long)MAXULONGLONG) +# else +# define PERL_UQUAD_MAX (~(unsigned long long)0) +# endif +# endif +# endif + +# ifndef PERL_UQUAD_MIN +# define PERL_UQUAD_MIN ((unsigned long long)0L) +# endif + +# ifndef PERL_QUAD_MAX +# ifdef LONGLONG_MAX +# define PERL_QUAD_MAX ((long long)LONGLONG_MAX) +# else +# ifdef MAXLONGLONG +# define PERL_QUAD_MAX ((long long)MAXLONGLONG) +# else +# define PERL_QUAD_MAX ((long long) (PERL_UQUAD_MAX >> 1)) +# endif +# endif +# endif + +# ifndef PERL_QUAD_MIN +# ifdef LONGLONG_MIN +# define PERL_QUAD_MIN ((long long)LONGLONG_MIN) +# else +# ifdef MINLONGLONG +# define PERL_QUAD_MIN ((long long)MINLONGLONG) +# else +# define PERL_QUAD_MIN (-PERL_QUAD_MAX - ((3 & -1) == 3)) +# endif +# endif +# endif +#endif + +/* This is based on code from 5.003 perl.h */ +#ifdef HAS_QUAD +# ifdef cray +#ifndef IVTYPE +# define IVTYPE int +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_INT_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_INT_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_UINT_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_UINT_MAX +#endif + +# ifdef INTSIZE +#ifndef IVSIZE +# define IVSIZE INTSIZE +#endif + +# endif +# else +# if defined(convex) || defined(uts) +#ifndef IVTYPE +# define IVTYPE long long +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_QUAD_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_QUAD_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_UQUAD_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_UQUAD_MAX +#endif + +# ifdef LONGLONGSIZE +#ifndef IVSIZE +# define IVSIZE LONGLONGSIZE +#endif + +# endif +# else +#ifndef IVTYPE +# define IVTYPE long +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_LONG_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_LONG_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_ULONG_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_ULONG_MAX +#endif + +# ifdef LONGSIZE +#ifndef IVSIZE +# define IVSIZE LONGSIZE +#endif + +# endif +# endif +# endif +#ifndef IVSIZE +# define IVSIZE 8 +#endif + +#ifndef PERL_QUAD_MIN +# define PERL_QUAD_MIN IV_MIN +#endif + +#ifndef PERL_QUAD_MAX +# define PERL_QUAD_MAX IV_MAX +#endif + +#ifndef PERL_UQUAD_MIN +# define PERL_UQUAD_MIN UV_MIN +#endif + +#ifndef PERL_UQUAD_MAX +# define PERL_UQUAD_MAX UV_MAX +#endif + +#else +#ifndef IVTYPE +# define IVTYPE long +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_LONG_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_LONG_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_ULONG_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_ULONG_MAX +#endif + +#endif + +#ifndef IVSIZE +# ifdef LONGSIZE +# define IVSIZE LONGSIZE +# else +# define IVSIZE 4 /* A bold guess, but the best we can make. */ +# endif +#endif +#ifndef UVTYPE +# define UVTYPE unsigned IVTYPE +#endif + +#ifndef UVSIZE +# define UVSIZE IVSIZE +#endif +#ifndef sv_setuv +# define sv_setuv(sv, uv) \ + STMT_START { \ + UV TeMpUv = uv; \ + if (TeMpUv <= IV_MAX) \ + sv_setiv(sv, TeMpUv); \ + else \ + sv_setnv(sv, (double)TeMpUv); \ + } STMT_END +#endif +#ifndef newSVuv +# define newSVuv(uv) ((uv) <= IV_MAX ? newSViv((IV)uv) : newSVnv((NV)uv)) +#endif +#ifndef sv_2uv +# define sv_2uv(sv) ((PL_Sv = (sv)), (UV) (SvNOK(PL_Sv) ? SvNV(PL_Sv) : sv_2nv(PL_Sv))) +#endif + +#ifndef SvUVX +# define SvUVX(sv) ((UV)SvIVX(sv)) +#endif + +#ifndef SvUVXx +# define SvUVXx(sv) SvUVX(sv) +#endif + +#ifndef SvUV +# define SvUV(sv) (SvIOK(sv) ? SvUVX(sv) : sv_2uv(sv)) +#endif + +#ifndef SvUVx +# define SvUVx(sv) ((PL_Sv = (sv)), SvUV(PL_Sv)) +#endif + +/* Hint: sv_uv + * Always use the SvUVx() macro instead of sv_uv(). + */ +#ifndef sv_uv +# define sv_uv(sv) SvUVx(sv) +#endif + +#if !defined(SvUOK) && defined(SvIOK_UV) +# define SvUOK(sv) SvIOK_UV(sv) +#endif +#ifndef XST_mUV +# define XST_mUV(i,v) (ST(i) = sv_2mortal(newSVuv(v)) ) +#endif + +#ifndef XSRETURN_UV +# define XSRETURN_UV(v) STMT_START { XST_mUV(0,v); XSRETURN(1); } STMT_END +#endif +#ifndef PUSHu +# define PUSHu(u) STMT_START { sv_setuv(TARG, (UV)(u)); PUSHTARG; } STMT_END +#endif + +#ifndef XPUSHu +# define XPUSHu(u) STMT_START { sv_setuv(TARG, (UV)(u)); XPUSHTARG; } STMT_END +#endif + +#ifdef HAS_MEMCMP +#ifndef memNE +# define memNE(s1,s2,l) (memcmp(s1,s2,l)) +#endif + +#ifndef memEQ +# define memEQ(s1,s2,l) (!memcmp(s1,s2,l)) +#endif + +#else +#ifndef memNE +# define memNE(s1,s2,l) (bcmp(s1,s2,l)) +#endif + +#ifndef memEQ +# define memEQ(s1,s2,l) (!bcmp(s1,s2,l)) +#endif + +#endif +#ifndef MoveD +# define MoveD(s,d,n,t) memmove((char*)(d),(char*)(s), (n) * sizeof(t)) +#endif + +#ifndef CopyD +# define CopyD(s,d,n,t) memcpy((char*)(d),(char*)(s), (n) * sizeof(t)) +#endif + +#ifdef HAS_MEMSET +#ifndef ZeroD +# define ZeroD(d,n,t) memzero((char*)(d), (n) * sizeof(t)) +#endif + +#else +#ifndef ZeroD +# define ZeroD(d,n,t) ((void)memzero((char*)(d), (n) * sizeof(t)), d) +#endif + +#endif +#ifndef PoisonWith +# define PoisonWith(d,n,t,b) (void)memset((char*)(d), (U8)(b), (n) * sizeof(t)) +#endif + +#ifndef PoisonNew +# define PoisonNew(d,n,t) PoisonWith(d,n,t,0xAB) +#endif + +#ifndef PoisonFree +# define PoisonFree(d,n,t) PoisonWith(d,n,t,0xEF) +#endif + +#ifndef Poison +# define Poison(d,n,t) PoisonFree(d,n,t) +#endif +#ifndef Newx +# define Newx(v,n,t) New(0,v,n,t) +#endif + +#ifndef Newxc +# define Newxc(v,n,t,c) Newc(0,v,n,t,c) +#endif + +#ifndef Newxz +# define Newxz(v,n,t) Newz(0,v,n,t) +#endif + +#ifndef PERL_UNUSED_DECL +# ifdef HASATTRIBUTE +# if (defined(__GNUC__) && defined(__cplusplus)) || defined(__INTEL_COMPILER) +# define PERL_UNUSED_DECL +# else +# define PERL_UNUSED_DECL __attribute__((unused)) +# endif +# else +# define PERL_UNUSED_DECL +# endif +#endif + +#ifndef PERL_UNUSED_ARG +# if defined(lint) && defined(S_SPLINT_S) /* www.splint.org */ +# include +# define PERL_UNUSED_ARG(x) NOTE(ARGUNUSED(x)) +# else +# define PERL_UNUSED_ARG(x) ((void)x) +# endif +#endif + +#ifndef PERL_UNUSED_VAR +# define PERL_UNUSED_VAR(x) ((void)x) +#endif + +#ifndef PERL_UNUSED_CONTEXT +# ifdef USE_ITHREADS +# define PERL_UNUSED_CONTEXT PERL_UNUSED_ARG(my_perl) +# else +# define PERL_UNUSED_CONTEXT +# endif +#endif +#ifndef NOOP +# define NOOP /*EMPTY*/(void)0 +#endif + +#ifndef dNOOP +# define dNOOP extern int /*@unused@*/ Perl___notused PERL_UNUSED_DECL +#endif + +#ifndef NVTYPE +# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) +# define NVTYPE long double +# else +# define NVTYPE double +# endif +typedef NVTYPE NV; +#endif + +#ifndef INT2PTR +# if (IVSIZE == PTRSIZE) && (UVSIZE == PTRSIZE) +# define PTRV UV +# define INT2PTR(any,d) (any)(d) +# else +# if PTRSIZE == LONGSIZE +# define PTRV unsigned long +# else +# define PTRV unsigned +# endif +# define INT2PTR(any,d) (any)(PTRV)(d) +# endif +#endif + +#ifndef PTR2ul +# if PTRSIZE == LONGSIZE +# define PTR2ul(p) (unsigned long)(p) +# else +# define PTR2ul(p) INT2PTR(unsigned long,p) +# endif +#endif +#ifndef PTR2nat +# define PTR2nat(p) (PTRV)(p) +#endif + +#ifndef NUM2PTR +# define NUM2PTR(any,d) (any)PTR2nat(d) +#endif + +#ifndef PTR2IV +# define PTR2IV(p) INT2PTR(IV,p) +#endif + +#ifndef PTR2UV +# define PTR2UV(p) INT2PTR(UV,p) +#endif + +#ifndef PTR2NV +# define PTR2NV(p) NUM2PTR(NV,p) +#endif + +#undef START_EXTERN_C +#undef END_EXTERN_C +#undef EXTERN_C +#ifdef __cplusplus +# define START_EXTERN_C extern "C" { +# define END_EXTERN_C } +# define EXTERN_C extern "C" +#else +# define START_EXTERN_C +# define END_EXTERN_C +# define EXTERN_C extern +#endif + +#if defined(PERL_GCC_PEDANTIC) +# ifndef PERL_GCC_BRACE_GROUPS_FORBIDDEN +# define PERL_GCC_BRACE_GROUPS_FORBIDDEN +# endif +#endif + +#if defined(__GNUC__) && !defined(PERL_GCC_BRACE_GROUPS_FORBIDDEN) && !defined(__cplusplus) +# ifndef PERL_USE_GCC_BRACE_GROUPS +# define PERL_USE_GCC_BRACE_GROUPS +# endif +#endif + +#undef STMT_START +#undef STMT_END +#ifdef PERL_USE_GCC_BRACE_GROUPS +# define STMT_START (void)( /* gcc supports ``({ STATEMENTS; })'' */ +# define STMT_END ) +#else +# if defined(VOIDFLAGS) && (VOIDFLAGS) && (defined(sun) || defined(__sun__)) && !defined(__GNUC__) +# define STMT_START if (1) +# define STMT_END else (void)0 +# else +# define STMT_START do +# define STMT_END while (0) +# endif +#endif +#ifndef boolSV +# define boolSV(b) ((b) ? &PL_sv_yes : &PL_sv_no) +#endif + +/* DEFSV appears first in 5.004_56 */ +#ifndef DEFSV +# define DEFSV GvSV(PL_defgv) +#endif + +#ifndef SAVE_DEFSV +# define SAVE_DEFSV SAVESPTR(GvSV(PL_defgv)) +#endif + +#ifndef DEFSV_set +# define DEFSV_set(sv) (DEFSV = (sv)) +#endif + +/* Older perls (<=5.003) lack AvFILLp */ +#ifndef AvFILLp +# define AvFILLp AvFILL +#endif +#ifndef ERRSV +# define ERRSV get_sv("@",FALSE) +#endif + +/* Hint: gv_stashpvn + * This function's backport doesn't support the length parameter, but + * rather ignores it. Portability can only be ensured if the length + * parameter is used for speed reasons, but the length can always be + * correctly computed from the string argument. + */ +#ifndef gv_stashpvn +# define gv_stashpvn(str,len,create) gv_stashpv(str,create) +#endif + +/* Replace: 1 */ +#ifndef get_cv +# define get_cv perl_get_cv +#endif + +#ifndef get_sv +# define get_sv perl_get_sv +#endif + +#ifndef get_av +# define get_av perl_get_av +#endif + +#ifndef get_hv +# define get_hv perl_get_hv +#endif + +/* Replace: 0 */ +#ifndef dUNDERBAR +# define dUNDERBAR dNOOP +#endif + +#ifndef UNDERBAR +# define UNDERBAR DEFSV +#endif +#ifndef dAX +# define dAX I32 ax = MARK - PL_stack_base + 1 +#endif + +#ifndef dITEMS +# define dITEMS I32 items = SP - MARK +#endif +#ifndef dXSTARG +# define dXSTARG SV * targ = sv_newmortal() +#endif +#ifndef dAXMARK +# define dAXMARK I32 ax = POPMARK; \ + register SV ** const mark = PL_stack_base + ax++ +#endif +#ifndef XSprePUSH +# define XSprePUSH (sp = PL_stack_base + ax - 1) +#endif + +#if (PERL_BCDVERSION < 0x5005000) +# undef XSRETURN +# define XSRETURN(off) \ + STMT_START { \ + PL_stack_sp = PL_stack_base + ax + ((off) - 1); \ + return; \ + } STMT_END +#endif +#ifndef PERL_ABS +# define PERL_ABS(x) ((x) < 0 ? -(x) : (x)) +#endif +#ifndef dVAR +# define dVAR dNOOP +#endif +#ifndef SVf +# define SVf "_" +#endif +#ifndef UTF8_MAXBYTES +# define UTF8_MAXBYTES UTF8_MAXLEN +#endif +#ifndef CPERLscope +# define CPERLscope(x) x +#endif +#ifndef PERL_HASH +# define PERL_HASH(hash,str,len) \ + STMT_START { \ + const char *s_PeRlHaSh = str; \ + I32 i_PeRlHaSh = len; \ + U32 hash_PeRlHaSh = 0; \ + while (i_PeRlHaSh--) \ + hash_PeRlHaSh = hash_PeRlHaSh * 33 + *s_PeRlHaSh++; \ + (hash) = hash_PeRlHaSh; \ + } STMT_END +#endif + +#ifndef PERLIO_FUNCS_DECL +# ifdef PERLIO_FUNCS_CONST +# define PERLIO_FUNCS_DECL(funcs) const PerlIO_funcs funcs +# define PERLIO_FUNCS_CAST(funcs) (PerlIO_funcs*)(funcs) +# else +# define PERLIO_FUNCS_DECL(funcs) PerlIO_funcs funcs +# define PERLIO_FUNCS_CAST(funcs) (funcs) +# endif +#endif + +/* provide these typedefs for older perls */ +#if (PERL_BCDVERSION < 0x5009003) + +# ifdef ARGSproto +typedef OP* (CPERLscope(*Perl_ppaddr_t))(ARGSproto); +# else +typedef OP* (CPERLscope(*Perl_ppaddr_t))(pTHX); +# endif + +typedef OP* (CPERLscope(*Perl_check_t)) (pTHX_ OP*); + +#endif +#ifndef isPSXSPC +# define isPSXSPC(c) (isSPACE(c) || (c) == '\v') +#endif + +#ifndef isBLANK +# define isBLANK(c) ((c) == ' ' || (c) == '\t') +#endif + +#ifdef EBCDIC +#ifndef isALNUMC +# define isALNUMC(c) isalnum(c) +#endif + +#ifndef isASCII +# define isASCII(c) isascii(c) +#endif + +#ifndef isCNTRL +# define isCNTRL(c) iscntrl(c) +#endif + +#ifndef isGRAPH +# define isGRAPH(c) isgraph(c) +#endif + +#ifndef isPRINT +# define isPRINT(c) isprint(c) +#endif + +#ifndef isPUNCT +# define isPUNCT(c) ispunct(c) +#endif + +#ifndef isXDIGIT +# define isXDIGIT(c) isxdigit(c) +#endif + +#else +# if (PERL_BCDVERSION < 0x5010000) +/* Hint: isPRINT + * The implementation in older perl versions includes all of the + * isSPACE() characters, which is wrong. The version provided by + * Devel::PPPort always overrides a present buggy version. + */ +# undef isPRINT +# endif +#ifndef isALNUMC +# define isALNUMC(c) (isALPHA(c) || isDIGIT(c)) +#endif + +#ifndef isASCII +# define isASCII(c) ((c) <= 127) +#endif + +#ifndef isCNTRL +# define isCNTRL(c) ((c) < ' ' || (c) == 127) +#endif + +#ifndef isGRAPH +# define isGRAPH(c) (isALNUM(c) || isPUNCT(c)) +#endif + +#ifndef isPRINT +# define isPRINT(c) (((c) >= 32 && (c) < 127)) +#endif + +#ifndef isPUNCT +# define isPUNCT(c) (((c) >= 33 && (c) <= 47) || ((c) >= 58 && (c) <= 64) || ((c) >= 91 && (c) <= 96) || ((c) >= 123 && (c) <= 126)) +#endif + +#ifndef isXDIGIT +# define isXDIGIT(c) (isDIGIT(c) || ((c) >= 'a' && (c) <= 'f') || ((c) >= 'A' && (c) <= 'F')) +#endif + +#endif + +#ifndef PERL_SIGNALS_UNSAFE_FLAG + +#define PERL_SIGNALS_UNSAFE_FLAG 0x0001 + +#if (PERL_BCDVERSION < 0x5008000) +# define D_PPP_PERL_SIGNALS_INIT PERL_SIGNALS_UNSAFE_FLAG +#else +# define D_PPP_PERL_SIGNALS_INIT 0 +#endif + +#if defined(NEED_PL_signals) +static U32 DPPP_(my_PL_signals) = D_PPP_PERL_SIGNALS_INIT; +#elif defined(NEED_PL_signals_GLOBAL) +U32 DPPP_(my_PL_signals) = D_PPP_PERL_SIGNALS_INIT; +#else +extern U32 DPPP_(my_PL_signals); +#endif +#define PL_signals DPPP_(my_PL_signals) + +#endif + +/* Hint: PL_ppaddr + * Calling an op via PL_ppaddr requires passing a context argument + * for threaded builds. Since the context argument is different for + * 5.005 perls, you can use aTHXR (supplied by ppport.h), which will + * automatically be defined as the correct argument. + */ + +#if (PERL_BCDVERSION <= 0x5005005) +/* Replace: 1 */ +# define PL_ppaddr ppaddr +# define PL_no_modify no_modify +/* Replace: 0 */ +#endif + +#if (PERL_BCDVERSION <= 0x5004005) +/* Replace: 1 */ +# define PL_DBsignal DBsignal +# define PL_DBsingle DBsingle +# define PL_DBsub DBsub +# define PL_DBtrace DBtrace +# define PL_Sv Sv +# define PL_bufend bufend +# define PL_bufptr bufptr +# define PL_compiling compiling +# define PL_copline copline +# define PL_curcop curcop +# define PL_curstash curstash +# define PL_debstash debstash +# define PL_defgv defgv +# define PL_diehook diehook +# define PL_dirty dirty +# define PL_dowarn dowarn +# define PL_errgv errgv +# define PL_expect expect +# define PL_hexdigit hexdigit +# define PL_hints hints +# define PL_laststatval laststatval +# define PL_lex_state lex_state +# define PL_lex_stuff lex_stuff +# define PL_linestr linestr +# define PL_na na +# define PL_perl_destruct_level perl_destruct_level +# define PL_perldb perldb +# define PL_rsfp_filters rsfp_filters +# define PL_rsfp rsfp +# define PL_stack_base stack_base +# define PL_stack_sp stack_sp +# define PL_statcache statcache +# define PL_stdingv stdingv +# define PL_sv_arenaroot sv_arenaroot +# define PL_sv_no sv_no +# define PL_sv_undef sv_undef +# define PL_sv_yes sv_yes +# define PL_tainted tainted +# define PL_tainting tainting +# define PL_tokenbuf tokenbuf +/* Replace: 0 */ +#endif + +/* Warning: PL_parser + * For perl versions earlier than 5.9.5, this is an always + * non-NULL dummy. Also, it cannot be dereferenced. Don't + * use it if you can avoid is and unless you absolutely know + * what you're doing. + * If you always check that PL_parser is non-NULL, you can + * define DPPP_PL_parser_NO_DUMMY to avoid the creation of + * a dummy parser structure. + */ + +#if (PERL_BCDVERSION >= 0x5009005) +# ifdef DPPP_PL_parser_NO_DUMMY +# define D_PPP_my_PL_parser_var(var) ((PL_parser ? PL_parser : \ + (croak("panic: PL_parser == NULL in %s:%d", \ + __FILE__, __LINE__), (yy_parser *) NULL))->var) +# else +# ifdef DPPP_PL_parser_NO_DUMMY_WARNING +# define D_PPP_parser_dummy_warning(var) +# else +# define D_PPP_parser_dummy_warning(var) \ + warn("warning: dummy PL_" #var " used in %s:%d", __FILE__, __LINE__), +# endif +# define D_PPP_my_PL_parser_var(var) ((PL_parser ? PL_parser : \ + (D_PPP_parser_dummy_warning(var) &DPPP_(dummy_PL_parser)))->var) +#if defined(NEED_PL_parser) +static yy_parser DPPP_(dummy_PL_parser); +#elif defined(NEED_PL_parser_GLOBAL) +yy_parser DPPP_(dummy_PL_parser); +#else +extern yy_parser DPPP_(dummy_PL_parser); +#endif + +# endif + +/* PL_expect, PL_copline, PL_rsfp, PL_rsfp_filters, PL_linestr, PL_bufptr, PL_bufend, PL_lex_state, PL_lex_stuff, PL_tokenbuf depends on PL_parser */ +/* Warning: PL_expect, PL_copline, PL_rsfp, PL_rsfp_filters, PL_linestr, PL_bufptr, PL_bufend, PL_lex_state, PL_lex_stuff, PL_tokenbuf + * Do not use this variable unless you know exactly what you're + * doint. It is internal to the perl parser and may change or even + * be removed in the future. As of perl 5.9.5, you have to check + * for (PL_parser != NULL) for this variable to have any effect. + * An always non-NULL PL_parser dummy is provided for earlier + * perl versions. + * If PL_parser is NULL when you try to access this variable, a + * dummy is being accessed instead and a warning is issued unless + * you define DPPP_PL_parser_NO_DUMMY_WARNING. + * If DPPP_PL_parser_NO_DUMMY is defined, the code trying to access + * this variable will croak with a panic message. + */ + +# define PL_expect D_PPP_my_PL_parser_var(expect) +# define PL_copline D_PPP_my_PL_parser_var(copline) +# define PL_rsfp D_PPP_my_PL_parser_var(rsfp) +# define PL_rsfp_filters D_PPP_my_PL_parser_var(rsfp_filters) +# define PL_linestr D_PPP_my_PL_parser_var(linestr) +# define PL_bufptr D_PPP_my_PL_parser_var(bufptr) +# define PL_bufend D_PPP_my_PL_parser_var(bufend) +# define PL_lex_state D_PPP_my_PL_parser_var(lex_state) +# define PL_lex_stuff D_PPP_my_PL_parser_var(lex_stuff) +# define PL_tokenbuf D_PPP_my_PL_parser_var(tokenbuf) + +#else + +/* ensure that PL_parser != NULL and cannot be dereferenced */ +# define PL_parser ((void *) 1) + +#endif +#ifndef mPUSHs +# define mPUSHs(s) PUSHs(sv_2mortal(s)) +#endif + +#ifndef PUSHmortal +# define PUSHmortal PUSHs(sv_newmortal()) +#endif + +#ifndef mPUSHp +# define mPUSHp(p,l) sv_setpvn(PUSHmortal, (p), (l)) +#endif + +#ifndef mPUSHn +# define mPUSHn(n) sv_setnv(PUSHmortal, (NV)(n)) +#endif + +#ifndef mPUSHi +# define mPUSHi(i) sv_setiv(PUSHmortal, (IV)(i)) +#endif + +#ifndef mPUSHu +# define mPUSHu(u) sv_setuv(PUSHmortal, (UV)(u)) +#endif +#ifndef mXPUSHs +# define mXPUSHs(s) XPUSHs(sv_2mortal(s)) +#endif + +#ifndef XPUSHmortal +# define XPUSHmortal XPUSHs(sv_newmortal()) +#endif + +#ifndef mXPUSHp +# define mXPUSHp(p,l) STMT_START { EXTEND(sp,1); sv_setpvn(PUSHmortal, (p), (l)); } STMT_END +#endif + +#ifndef mXPUSHn +# define mXPUSHn(n) STMT_START { EXTEND(sp,1); sv_setnv(PUSHmortal, (NV)(n)); } STMT_END +#endif + +#ifndef mXPUSHi +# define mXPUSHi(i) STMT_START { EXTEND(sp,1); sv_setiv(PUSHmortal, (IV)(i)); } STMT_END +#endif + +#ifndef mXPUSHu +# define mXPUSHu(u) STMT_START { EXTEND(sp,1); sv_setuv(PUSHmortal, (UV)(u)); } STMT_END +#endif + +/* Replace: 1 */ +#ifndef call_sv +# define call_sv perl_call_sv +#endif + +#ifndef call_pv +# define call_pv perl_call_pv +#endif + +#ifndef call_argv +# define call_argv perl_call_argv +#endif + +#ifndef call_method +# define call_method perl_call_method +#endif +#ifndef eval_sv +# define eval_sv perl_eval_sv +#endif + +/* Replace: 0 */ +#ifndef PERL_LOADMOD_DENY +# define PERL_LOADMOD_DENY 0x1 +#endif + +#ifndef PERL_LOADMOD_NOIMPORT +# define PERL_LOADMOD_NOIMPORT 0x2 +#endif + +#ifndef PERL_LOADMOD_IMPORT_OPS +# define PERL_LOADMOD_IMPORT_OPS 0x4 +#endif + +#ifndef G_METHOD +# define G_METHOD 64 +# ifdef call_sv +# undef call_sv +# endif +# if (PERL_BCDVERSION < 0x5006000) +# define call_sv(sv, flags) ((flags) & G_METHOD ? perl_call_method((char *) SvPV_nolen_const(sv), \ + (flags) & ~G_METHOD) : perl_call_sv(sv, flags)) +# else +# define call_sv(sv, flags) ((flags) & G_METHOD ? Perl_call_method(aTHX_ (char *) SvPV_nolen_const(sv), \ + (flags) & ~G_METHOD) : Perl_call_sv(aTHX_ sv, flags)) +# endif +#endif + +/* Replace perl_eval_pv with eval_pv */ + +#ifndef eval_pv +#if defined(NEED_eval_pv) +static SV* DPPP_(my_eval_pv)(char *p, I32 croak_on_error); +static +#else +extern SV* DPPP_(my_eval_pv)(char *p, I32 croak_on_error); +#endif + +#ifdef eval_pv +# undef eval_pv +#endif +#define eval_pv(a,b) DPPP_(my_eval_pv)(aTHX_ a,b) +#define Perl_eval_pv DPPP_(my_eval_pv) + +#if defined(NEED_eval_pv) || defined(NEED_eval_pv_GLOBAL) + +SV* +DPPP_(my_eval_pv)(char *p, I32 croak_on_error) +{ + dSP; + SV* sv = newSVpv(p, 0); + + PUSHMARK(sp); + eval_sv(sv, G_SCALAR); + SvREFCNT_dec(sv); + + SPAGAIN; + sv = POPs; + PUTBACK; + + if (croak_on_error && SvTRUE(GvSV(errgv))) + croak(SvPVx(GvSV(errgv), na)); + + return sv; +} + +#endif +#endif + +#ifndef vload_module +#if defined(NEED_vload_module) +static void DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args); +static +#else +extern void DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args); +#endif + +#ifdef vload_module +# undef vload_module +#endif +#define vload_module(a,b,c,d) DPPP_(my_vload_module)(aTHX_ a,b,c,d) +#define Perl_vload_module DPPP_(my_vload_module) + +#if defined(NEED_vload_module) || defined(NEED_vload_module_GLOBAL) + +void +DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args) +{ + dTHR; + dVAR; + OP *veop, *imop; + + OP * const modname = newSVOP(OP_CONST, 0, name); + /* 5.005 has a somewhat hacky force_normal that doesn't croak on + SvREADONLY() if PL_compling is true. Current perls take care in + ck_require() to correctly turn off SvREADONLY before calling + force_normal_flags(). This seems a better fix than fudging PL_compling + */ + SvREADONLY_off(((SVOP*)modname)->op_sv); + modname->op_private |= OPpCONST_BARE; + if (ver) { + veop = newSVOP(OP_CONST, 0, ver); + } + else + veop = NULL; + if (flags & PERL_LOADMOD_NOIMPORT) { + imop = sawparens(newNULLLIST()); + } + else if (flags & PERL_LOADMOD_IMPORT_OPS) { + imop = va_arg(*args, OP*); + } + else { + SV *sv; + imop = NULL; + sv = va_arg(*args, SV*); + while (sv) { + imop = append_elem(OP_LIST, imop, newSVOP(OP_CONST, 0, sv)); + sv = va_arg(*args, SV*); + } + } + { + const line_t ocopline = PL_copline; + COP * const ocurcop = PL_curcop; + const int oexpect = PL_expect; + +#if (PERL_BCDVERSION >= 0x5004000) + utilize(!(flags & PERL_LOADMOD_DENY), start_subparse(FALSE, 0), + veop, modname, imop); +#else + utilize(!(flags & PERL_LOADMOD_DENY), start_subparse(), + modname, imop); +#endif + PL_expect = oexpect; + PL_copline = ocopline; + PL_curcop = ocurcop; + } +} + +#endif +#endif + +#ifndef load_module +#if defined(NEED_load_module) +static void DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...); +static +#else +extern void DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...); +#endif + +#ifdef load_module +# undef load_module +#endif +#define load_module DPPP_(my_load_module) +#define Perl_load_module DPPP_(my_load_module) + +#if defined(NEED_load_module) || defined(NEED_load_module_GLOBAL) + +void +DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...) +{ + va_list args; + va_start(args, ver); + vload_module(flags, name, ver, &args); + va_end(args); +} + +#endif +#endif +#ifndef newRV_inc +# define newRV_inc(sv) newRV(sv) /* Replace */ +#endif + +#ifndef newRV_noinc +#if defined(NEED_newRV_noinc) +static SV * DPPP_(my_newRV_noinc)(SV *sv); +static +#else +extern SV * DPPP_(my_newRV_noinc)(SV *sv); +#endif + +#ifdef newRV_noinc +# undef newRV_noinc +#endif +#define newRV_noinc(a) DPPP_(my_newRV_noinc)(aTHX_ a) +#define Perl_newRV_noinc DPPP_(my_newRV_noinc) + +#if defined(NEED_newRV_noinc) || defined(NEED_newRV_noinc_GLOBAL) +SV * +DPPP_(my_newRV_noinc)(SV *sv) +{ + SV *rv = (SV *)newRV(sv); + SvREFCNT_dec(sv); + return rv; +} +#endif +#endif + +/* Hint: newCONSTSUB + * Returns a CV* as of perl-5.7.1. This return value is not supported + * by Devel::PPPort. + */ + +/* newCONSTSUB from IO.xs is in the core starting with 5.004_63 */ +#if (PERL_BCDVERSION < 0x5004063) && (PERL_BCDVERSION != 0x5004005) +#if defined(NEED_newCONSTSUB) +static void DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv); +static +#else +extern void DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv); +#endif + +#ifdef newCONSTSUB +# undef newCONSTSUB +#endif +#define newCONSTSUB(a,b,c) DPPP_(my_newCONSTSUB)(aTHX_ a,b,c) +#define Perl_newCONSTSUB DPPP_(my_newCONSTSUB) + +#if defined(NEED_newCONSTSUB) || defined(NEED_newCONSTSUB_GLOBAL) + +/* This is just a trick to avoid a dependency of newCONSTSUB on PL_parser */ +/* (There's no PL_parser in perl < 5.005, so this is completely safe) */ +#define D_PPP_PL_copline PL_copline + +void +DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv) +{ + U32 oldhints = PL_hints; + HV *old_cop_stash = PL_curcop->cop_stash; + HV *old_curstash = PL_curstash; + line_t oldline = PL_curcop->cop_line; + PL_curcop->cop_line = D_PPP_PL_copline; + + PL_hints &= ~HINT_BLOCK_SCOPE; + if (stash) + PL_curstash = PL_curcop->cop_stash = stash; + + newSUB( + +#if (PERL_BCDVERSION < 0x5003022) + start_subparse(), +#elif (PERL_BCDVERSION == 0x5003022) + start_subparse(0), +#else /* 5.003_23 onwards */ + start_subparse(FALSE, 0), +#endif + + newSVOP(OP_CONST, 0, newSVpv((char *) name, 0)), + newSVOP(OP_CONST, 0, &PL_sv_no), /* SvPV(&PL_sv_no) == "" -- GMB */ + newSTATEOP(0, Nullch, newSVOP(OP_CONST, 0, sv)) + ); + + PL_hints = oldhints; + PL_curcop->cop_stash = old_cop_stash; + PL_curstash = old_curstash; + PL_curcop->cop_line = oldline; +} +#endif +#endif + +/* + * Boilerplate macros for initializing and accessing interpreter-local + * data from C. All statics in extensions should be reworked to use + * this, if you want to make the extension thread-safe. See ext/re/re.xs + * for an example of the use of these macros. + * + * Code that uses these macros is responsible for the following: + * 1. #define MY_CXT_KEY to a unique string, e.g. "DynaLoader_guts" + * 2. Declare a typedef named my_cxt_t that is a structure that contains + * all the data that needs to be interpreter-local. + * 3. Use the START_MY_CXT macro after the declaration of my_cxt_t. + * 4. Use the MY_CXT_INIT macro such that it is called exactly once + * (typically put in the BOOT: section). + * 5. Use the members of the my_cxt_t structure everywhere as + * MY_CXT.member. + * 6. Use the dMY_CXT macro (a declaration) in all the functions that + * access MY_CXT. + */ + +#if defined(MULTIPLICITY) || defined(PERL_OBJECT) || \ + defined(PERL_CAPI) || defined(PERL_IMPLICIT_CONTEXT) + +#ifndef START_MY_CXT + +/* This must appear in all extensions that define a my_cxt_t structure, + * right after the definition (i.e. at file scope). The non-threads + * case below uses it to declare the data as static. */ +#define START_MY_CXT + +#if (PERL_BCDVERSION < 0x5004068) +/* Fetches the SV that keeps the per-interpreter data. */ +#define dMY_CXT_SV \ + SV *my_cxt_sv = get_sv(MY_CXT_KEY, FALSE) +#else /* >= perl5.004_68 */ +#define dMY_CXT_SV \ + SV *my_cxt_sv = *hv_fetch(PL_modglobal, MY_CXT_KEY, \ + sizeof(MY_CXT_KEY)-1, TRUE) +#endif /* < perl5.004_68 */ + +/* This declaration should be used within all functions that use the + * interpreter-local data. */ +#define dMY_CXT \ + dMY_CXT_SV; \ + my_cxt_t *my_cxtp = INT2PTR(my_cxt_t*,SvUV(my_cxt_sv)) + +/* Creates and zeroes the per-interpreter data. + * (We allocate my_cxtp in a Perl SV so that it will be released when + * the interpreter goes away.) */ +#define MY_CXT_INIT \ + dMY_CXT_SV; \ + /* newSV() allocates one more than needed */ \ + my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ + Zero(my_cxtp, 1, my_cxt_t); \ + sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) + +/* This macro must be used to access members of the my_cxt_t structure. + * e.g. MYCXT.some_data */ +#define MY_CXT (*my_cxtp) + +/* Judicious use of these macros can reduce the number of times dMY_CXT + * is used. Use is similar to pTHX, aTHX etc. */ +#define pMY_CXT my_cxt_t *my_cxtp +#define pMY_CXT_ pMY_CXT, +#define _pMY_CXT ,pMY_CXT +#define aMY_CXT my_cxtp +#define aMY_CXT_ aMY_CXT, +#define _aMY_CXT ,aMY_CXT + +#endif /* START_MY_CXT */ + +#ifndef MY_CXT_CLONE +/* Clones the per-interpreter data. */ +#define MY_CXT_CLONE \ + dMY_CXT_SV; \ + my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ + Copy(INT2PTR(my_cxt_t*, SvUV(my_cxt_sv)), my_cxtp, 1, my_cxt_t);\ + sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) +#endif + +#else /* single interpreter */ + +#ifndef START_MY_CXT + +#define START_MY_CXT static my_cxt_t my_cxt; +#define dMY_CXT_SV dNOOP +#define dMY_CXT dNOOP +#define MY_CXT_INIT NOOP +#define MY_CXT my_cxt + +#define pMY_CXT void +#define pMY_CXT_ +#define _pMY_CXT +#define aMY_CXT +#define aMY_CXT_ +#define _aMY_CXT + +#endif /* START_MY_CXT */ + +#ifndef MY_CXT_CLONE +#define MY_CXT_CLONE NOOP +#endif + +#endif + +#ifndef IVdf +# if IVSIZE == LONGSIZE +# define IVdf "ld" +# define UVuf "lu" +# define UVof "lo" +# define UVxf "lx" +# define UVXf "lX" +# else +# if IVSIZE == INTSIZE +# define IVdf "d" +# define UVuf "u" +# define UVof "o" +# define UVxf "x" +# define UVXf "X" +# endif +# endif +#endif + +#ifndef NVef +# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) && \ + defined(PERL_PRIfldbl) && (PERL_BCDVERSION != 0x5006000) + /* Not very likely, but let's try anyway. */ +# define NVef PERL_PRIeldbl +# define NVff PERL_PRIfldbl +# define NVgf PERL_PRIgldbl +# else +# define NVef "e" +# define NVff "f" +# define NVgf "g" +# endif +#endif + +#ifndef SvREFCNT_inc +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc(sv) \ + ({ \ + SV * const _sv = (SV*)(sv); \ + if (_sv) \ + (SvREFCNT(_sv))++; \ + _sv; \ + }) +# else +# define SvREFCNT_inc(sv) \ + ((PL_Sv=(SV*)(sv)) ? (++(SvREFCNT(PL_Sv)),PL_Sv) : NULL) +# endif +#endif + +#ifndef SvREFCNT_inc_simple +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc_simple(sv) \ + ({ \ + if (sv) \ + (SvREFCNT(sv))++; \ + (SV *)(sv); \ + }) +# else +# define SvREFCNT_inc_simple(sv) \ + ((sv) ? (SvREFCNT(sv)++,(SV*)(sv)) : NULL) +# endif +#endif + +#ifndef SvREFCNT_inc_NN +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc_NN(sv) \ + ({ \ + SV * const _sv = (SV*)(sv); \ + SvREFCNT(_sv)++; \ + _sv; \ + }) +# else +# define SvREFCNT_inc_NN(sv) \ + (PL_Sv=(SV*)(sv),++(SvREFCNT(PL_Sv)),PL_Sv) +# endif +#endif + +#ifndef SvREFCNT_inc_void +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc_void(sv) \ + ({ \ + SV * const _sv = (SV*)(sv); \ + if (_sv) \ + (void)(SvREFCNT(_sv)++); \ + }) +# else +# define SvREFCNT_inc_void(sv) \ + (void)((PL_Sv=(SV*)(sv)) ? ++(SvREFCNT(PL_Sv)) : 0) +# endif +#endif +#ifndef SvREFCNT_inc_simple_void +# define SvREFCNT_inc_simple_void(sv) STMT_START { if (sv) SvREFCNT(sv)++; } STMT_END +#endif + +#ifndef SvREFCNT_inc_simple_NN +# define SvREFCNT_inc_simple_NN(sv) (++SvREFCNT(sv), (SV*)(sv)) +#endif + +#ifndef SvREFCNT_inc_void_NN +# define SvREFCNT_inc_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) +#endif + +#ifndef SvREFCNT_inc_simple_void_NN +# define SvREFCNT_inc_simple_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) +#endif + +#if (PERL_BCDVERSION < 0x5006000) +# define D_PPP_CONSTPV_ARG(x) ((char *) (x)) +#else +# define D_PPP_CONSTPV_ARG(x) (x) +#endif +#ifndef newSVpvn +# define newSVpvn(data,len) ((data) \ + ? ((len) ? newSVpv((data), (len)) : newSVpv("", 0)) \ + : newSV(0)) +#endif +#ifndef newSVpvn_utf8 +# define newSVpvn_utf8(s, len, u) newSVpvn_flags((s), (len), (u) ? SVf_UTF8 : 0) +#endif +#ifndef SVf_UTF8 +# define SVf_UTF8 0 +#endif + +#ifndef newSVpvn_flags + +#if defined(NEED_newSVpvn_flags) +static SV * DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags); +static +#else +extern SV * DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags); +#endif + +#ifdef newSVpvn_flags +# undef newSVpvn_flags +#endif +#define newSVpvn_flags(a,b,c) DPPP_(my_newSVpvn_flags)(aTHX_ a,b,c) +#define Perl_newSVpvn_flags DPPP_(my_newSVpvn_flags) + +#if defined(NEED_newSVpvn_flags) || defined(NEED_newSVpvn_flags_GLOBAL) + +SV * +DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags) +{ + SV *sv = newSVpvn(D_PPP_CONSTPV_ARG(s), len); + SvFLAGS(sv) |= (flags & SVf_UTF8); + return (flags & SVs_TEMP) ? sv_2mortal(sv) : sv; +} + +#endif + +#endif + +/* Backwards compatibility stuff... :-( */ +#if !defined(NEED_sv_2pv_flags) && defined(NEED_sv_2pv_nolen) +# define NEED_sv_2pv_flags +#endif +#if !defined(NEED_sv_2pv_flags_GLOBAL) && defined(NEED_sv_2pv_nolen_GLOBAL) +# define NEED_sv_2pv_flags_GLOBAL +#endif + +/* Hint: sv_2pv_nolen + * Use the SvPV_nolen() or SvPV_nolen_const() macros instead of sv_2pv_nolen(). + */ +#ifndef sv_2pv_nolen +# define sv_2pv_nolen(sv) SvPV_nolen(sv) +#endif + +#ifdef SvPVbyte + +/* Hint: SvPVbyte + * Does not work in perl-5.6.1, ppport.h implements a version + * borrowed from perl-5.7.3. + */ + +#if (PERL_BCDVERSION < 0x5007000) + +#if defined(NEED_sv_2pvbyte) +static char * DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp); +static +#else +extern char * DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp); +#endif + +#ifdef sv_2pvbyte +# undef sv_2pvbyte +#endif +#define sv_2pvbyte(a,b) DPPP_(my_sv_2pvbyte)(aTHX_ a,b) +#define Perl_sv_2pvbyte DPPP_(my_sv_2pvbyte) + +#if defined(NEED_sv_2pvbyte) || defined(NEED_sv_2pvbyte_GLOBAL) + +char * +DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp) +{ + sv_utf8_downgrade(sv,0); + return SvPV(sv,*lp); +} + +#endif + +/* Hint: sv_2pvbyte + * Use the SvPVbyte() macro instead of sv_2pvbyte(). + */ + +#undef SvPVbyte + +#define SvPVbyte(sv, lp) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_UTF8)) == (SVf_POK) \ + ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_2pvbyte(sv, &lp)) + +#endif + +#else + +# define SvPVbyte SvPV +# define sv_2pvbyte sv_2pv + +#endif +#ifndef sv_2pvbyte_nolen +# define sv_2pvbyte_nolen(sv) sv_2pv_nolen(sv) +#endif + +/* Hint: sv_pvn + * Always use the SvPV() macro instead of sv_pvn(). + */ + +/* Hint: sv_pvn_force + * Always use the SvPV_force() macro instead of sv_pvn_force(). + */ + +/* If these are undefined, they're not handled by the core anyway */ +#ifndef SV_IMMEDIATE_UNREF +# define SV_IMMEDIATE_UNREF 0 +#endif + +#ifndef SV_GMAGIC +# define SV_GMAGIC 0 +#endif + +#ifndef SV_COW_DROP_PV +# define SV_COW_DROP_PV 0 +#endif + +#ifndef SV_UTF8_NO_ENCODING +# define SV_UTF8_NO_ENCODING 0 +#endif + +#ifndef SV_NOSTEAL +# define SV_NOSTEAL 0 +#endif + +#ifndef SV_CONST_RETURN +# define SV_CONST_RETURN 0 +#endif + +#ifndef SV_MUTABLE_RETURN +# define SV_MUTABLE_RETURN 0 +#endif + +#ifndef SV_SMAGIC +# define SV_SMAGIC 0 +#endif + +#ifndef SV_HAS_TRAILING_NUL +# define SV_HAS_TRAILING_NUL 0 +#endif + +#ifndef SV_COW_SHARED_HASH_KEYS +# define SV_COW_SHARED_HASH_KEYS 0 +#endif + +#if (PERL_BCDVERSION < 0x5007002) + +#if defined(NEED_sv_2pv_flags) +static char * DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +static +#else +extern char * DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +#endif + +#ifdef sv_2pv_flags +# undef sv_2pv_flags +#endif +#define sv_2pv_flags(a,b,c) DPPP_(my_sv_2pv_flags)(aTHX_ a,b,c) +#define Perl_sv_2pv_flags DPPP_(my_sv_2pv_flags) + +#if defined(NEED_sv_2pv_flags) || defined(NEED_sv_2pv_flags_GLOBAL) + +char * +DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags) +{ + STRLEN n_a = (STRLEN) flags; + return sv_2pv(sv, lp ? lp : &n_a); +} + +#endif + +#if defined(NEED_sv_pvn_force_flags) +static char * DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +static +#else +extern char * DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +#endif + +#ifdef sv_pvn_force_flags +# undef sv_pvn_force_flags +#endif +#define sv_pvn_force_flags(a,b,c) DPPP_(my_sv_pvn_force_flags)(aTHX_ a,b,c) +#define Perl_sv_pvn_force_flags DPPP_(my_sv_pvn_force_flags) + +#if defined(NEED_sv_pvn_force_flags) || defined(NEED_sv_pvn_force_flags_GLOBAL) + +char * +DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags) +{ + STRLEN n_a = (STRLEN) flags; + return sv_pvn_force(sv, lp ? lp : &n_a); +} + +#endif + +#endif + +#if (PERL_BCDVERSION < 0x5008008) || ( (PERL_BCDVERSION >= 0x5009000) && (PERL_BCDVERSION < 0x5009003) ) +# define DPPP_SVPV_NOLEN_LP_ARG &PL_na +#else +# define DPPP_SVPV_NOLEN_LP_ARG 0 +#endif +#ifndef SvPV_const +# define SvPV_const(sv, lp) SvPV_flags_const(sv, lp, SV_GMAGIC) +#endif + +#ifndef SvPV_mutable +# define SvPV_mutable(sv, lp) SvPV_flags_mutable(sv, lp, SV_GMAGIC) +#endif +#ifndef SvPV_flags +# define SvPV_flags(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_2pv_flags(sv, &lp, flags)) +#endif +#ifndef SvPV_flags_const +# define SvPV_flags_const(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX_const(sv)) : \ + (const char*) sv_2pv_flags(sv, &lp, flags|SV_CONST_RETURN)) +#endif +#ifndef SvPV_flags_const_nolen +# define SvPV_flags_const_nolen(sv, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? SvPVX_const(sv) : \ + (const char*) sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, flags|SV_CONST_RETURN)) +#endif +#ifndef SvPV_flags_mutable +# define SvPV_flags_mutable(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX_mutable(sv)) : \ + sv_2pv_flags(sv, &lp, flags|SV_MUTABLE_RETURN)) +#endif +#ifndef SvPV_force +# define SvPV_force(sv, lp) SvPV_force_flags(sv, lp, SV_GMAGIC) +#endif + +#ifndef SvPV_force_nolen +# define SvPV_force_nolen(sv) SvPV_force_flags_nolen(sv, SV_GMAGIC) +#endif + +#ifndef SvPV_force_mutable +# define SvPV_force_mutable(sv, lp) SvPV_force_flags_mutable(sv, lp, SV_GMAGIC) +#endif + +#ifndef SvPV_force_nomg +# define SvPV_force_nomg(sv, lp) SvPV_force_flags(sv, lp, 0) +#endif + +#ifndef SvPV_force_nomg_nolen +# define SvPV_force_nomg_nolen(sv) SvPV_force_flags_nolen(sv, 0) +#endif +#ifndef SvPV_force_flags +# define SvPV_force_flags(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_pvn_force_flags(sv, &lp, flags)) +#endif +#ifndef SvPV_force_flags_nolen +# define SvPV_force_flags_nolen(sv, flags) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ + ? SvPVX(sv) : sv_pvn_force_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, flags)) +#endif +#ifndef SvPV_force_flags_mutable +# define SvPV_force_flags_mutable(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX_mutable(sv)) \ + : sv_pvn_force_flags(sv, &lp, flags|SV_MUTABLE_RETURN)) +#endif +#ifndef SvPV_nolen +# define SvPV_nolen(sv) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? SvPVX(sv) : sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, SV_GMAGIC)) +#endif +#ifndef SvPV_nolen_const +# define SvPV_nolen_const(sv) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? SvPVX_const(sv) : sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, SV_GMAGIC|SV_CONST_RETURN)) +#endif +#ifndef SvPV_nomg +# define SvPV_nomg(sv, lp) SvPV_flags(sv, lp, 0) +#endif + +#ifndef SvPV_nomg_const +# define SvPV_nomg_const(sv, lp) SvPV_flags_const(sv, lp, 0) +#endif + +#ifndef SvPV_nomg_const_nolen +# define SvPV_nomg_const_nolen(sv) SvPV_flags_const_nolen(sv, 0) +#endif +#ifndef SvPV_renew +# define SvPV_renew(sv,n) STMT_START { SvLEN_set(sv, n); \ + SvPV_set((sv), (char *) saferealloc( \ + (Malloc_t)SvPVX(sv), (MEM_SIZE)((n)))); \ + } STMT_END +#endif +#ifndef SvMAGIC_set +# define SvMAGIC_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_PVMG); \ + (((XPVMG*) SvANY(sv))->xmg_magic = (val)); } STMT_END +#endif + +#if (PERL_BCDVERSION < 0x5009003) +#ifndef SvPVX_const +# define SvPVX_const(sv) ((const char*) (0 + SvPVX(sv))) +#endif + +#ifndef SvPVX_mutable +# define SvPVX_mutable(sv) (0 + SvPVX(sv)) +#endif +#ifndef SvRV_set +# define SvRV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_RV); \ + (((XRV*) SvANY(sv))->xrv_rv = (val)); } STMT_END +#endif + +#else +#ifndef SvPVX_const +# define SvPVX_const(sv) ((const char*)((sv)->sv_u.svu_pv)) +#endif + +#ifndef SvPVX_mutable +# define SvPVX_mutable(sv) ((sv)->sv_u.svu_pv) +#endif +#ifndef SvRV_set +# define SvRV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_RV); \ + ((sv)->sv_u.svu_rv = (val)); } STMT_END +#endif + +#endif +#ifndef SvSTASH_set +# define SvSTASH_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_PVMG); \ + (((XPVMG*) SvANY(sv))->xmg_stash = (val)); } STMT_END +#endif + +#if (PERL_BCDVERSION < 0x5004000) +#ifndef SvUV_set +# define SvUV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) == SVt_IV || SvTYPE(sv) >= SVt_PVIV); \ + (((XPVIV*) SvANY(sv))->xiv_iv = (IV) (val)); } STMT_END +#endif + +#else +#ifndef SvUV_set +# define SvUV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) == SVt_IV || SvTYPE(sv) >= SVt_PVIV); \ + (((XPVUV*) SvANY(sv))->xuv_uv = (val)); } STMT_END +#endif + +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(vnewSVpvf) +#if defined(NEED_vnewSVpvf) +static SV * DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args); +static +#else +extern SV * DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args); +#endif + +#ifdef vnewSVpvf +# undef vnewSVpvf +#endif +#define vnewSVpvf(a,b) DPPP_(my_vnewSVpvf)(aTHX_ a,b) +#define Perl_vnewSVpvf DPPP_(my_vnewSVpvf) + +#if defined(NEED_vnewSVpvf) || defined(NEED_vnewSVpvf_GLOBAL) + +SV * +DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args) +{ + register SV *sv = newSV(0); + sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); + return sv; +} + +#endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vcatpvf) +# define sv_vcatpvf(sv, pat, args) sv_vcatpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)) +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vsetpvf) +# define sv_vsetpvf(sv, pat, args) sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)) +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_catpvf_mg) +#if defined(NEED_sv_catpvf_mg) +static void DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +#endif + +#define Perl_sv_catpvf_mg DPPP_(my_sv_catpvf_mg) + +#if defined(NEED_sv_catpvf_mg) || defined(NEED_sv_catpvf_mg_GLOBAL) + +void +DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...) +{ + va_list args; + va_start(args, pat); + sv_vcatpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif + +#ifdef PERL_IMPLICIT_CONTEXT +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_catpvf_mg_nocontext) +#if defined(NEED_sv_catpvf_mg_nocontext) +static void DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...); +#endif + +#define sv_catpvf_mg_nocontext DPPP_(my_sv_catpvf_mg_nocontext) +#define Perl_sv_catpvf_mg_nocontext DPPP_(my_sv_catpvf_mg_nocontext) + +#if defined(NEED_sv_catpvf_mg_nocontext) || defined(NEED_sv_catpvf_mg_nocontext_GLOBAL) + +void +DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...) +{ + dTHX; + va_list args; + va_start(args, pat); + sv_vcatpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif +#endif + +/* sv_catpvf_mg depends on sv_catpvf_mg_nocontext */ +#ifndef sv_catpvf_mg +# ifdef PERL_IMPLICIT_CONTEXT +# define sv_catpvf_mg Perl_sv_catpvf_mg_nocontext +# else +# define sv_catpvf_mg Perl_sv_catpvf_mg +# endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vcatpvf_mg) +# define sv_vcatpvf_mg(sv, pat, args) \ + STMT_START { \ + sv_vcatpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); \ + SvSETMAGIC(sv); \ + } STMT_END +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_setpvf_mg) +#if defined(NEED_sv_setpvf_mg) +static void DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +#endif + +#define Perl_sv_setpvf_mg DPPP_(my_sv_setpvf_mg) + +#if defined(NEED_sv_setpvf_mg) || defined(NEED_sv_setpvf_mg_GLOBAL) + +void +DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...) +{ + va_list args; + va_start(args, pat); + sv_vsetpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif + +#ifdef PERL_IMPLICIT_CONTEXT +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_setpvf_mg_nocontext) +#if defined(NEED_sv_setpvf_mg_nocontext) +static void DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...); +#endif + +#define sv_setpvf_mg_nocontext DPPP_(my_sv_setpvf_mg_nocontext) +#define Perl_sv_setpvf_mg_nocontext DPPP_(my_sv_setpvf_mg_nocontext) + +#if defined(NEED_sv_setpvf_mg_nocontext) || defined(NEED_sv_setpvf_mg_nocontext_GLOBAL) + +void +DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...) +{ + dTHX; + va_list args; + va_start(args, pat); + sv_vsetpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif +#endif + +/* sv_setpvf_mg depends on sv_setpvf_mg_nocontext */ +#ifndef sv_setpvf_mg +# ifdef PERL_IMPLICIT_CONTEXT +# define sv_setpvf_mg Perl_sv_setpvf_mg_nocontext +# else +# define sv_setpvf_mg Perl_sv_setpvf_mg +# endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vsetpvf_mg) +# define sv_vsetpvf_mg(sv, pat, args) \ + STMT_START { \ + sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); \ + SvSETMAGIC(sv); \ + } STMT_END +#endif + +#ifndef newSVpvn_share + +#if defined(NEED_newSVpvn_share) +static SV * DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash); +static +#else +extern SV * DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash); +#endif + +#ifdef newSVpvn_share +# undef newSVpvn_share +#endif +#define newSVpvn_share(a,b,c) DPPP_(my_newSVpvn_share)(aTHX_ a,b,c) +#define Perl_newSVpvn_share DPPP_(my_newSVpvn_share) + +#if defined(NEED_newSVpvn_share) || defined(NEED_newSVpvn_share_GLOBAL) + +SV * +DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash) +{ + SV *sv; + if (len < 0) + len = -len; + if (!hash) + PERL_HASH(hash, (char*) src, len); + sv = newSVpvn((char *) src, len); + sv_upgrade(sv, SVt_PVIV); + SvIVX(sv) = hash; + SvREADONLY_on(sv); + SvPOK_on(sv); + return sv; +} + +#endif + +#endif +#ifndef SvSHARED_HASH +# define SvSHARED_HASH(sv) (0 + SvUVX(sv)) +#endif +#ifndef WARN_ALL +# define WARN_ALL 0 +#endif + +#ifndef WARN_CLOSURE +# define WARN_CLOSURE 1 +#endif + +#ifndef WARN_DEPRECATED +# define WARN_DEPRECATED 2 +#endif + +#ifndef WARN_EXITING +# define WARN_EXITING 3 +#endif + +#ifndef WARN_GLOB +# define WARN_GLOB 4 +#endif + +#ifndef WARN_IO +# define WARN_IO 5 +#endif + +#ifndef WARN_CLOSED +# define WARN_CLOSED 6 +#endif + +#ifndef WARN_EXEC +# define WARN_EXEC 7 +#endif + +#ifndef WARN_LAYER +# define WARN_LAYER 8 +#endif + +#ifndef WARN_NEWLINE +# define WARN_NEWLINE 9 +#endif + +#ifndef WARN_PIPE +# define WARN_PIPE 10 +#endif + +#ifndef WARN_UNOPENED +# define WARN_UNOPENED 11 +#endif + +#ifndef WARN_MISC +# define WARN_MISC 12 +#endif + +#ifndef WARN_NUMERIC +# define WARN_NUMERIC 13 +#endif + +#ifndef WARN_ONCE +# define WARN_ONCE 14 +#endif + +#ifndef WARN_OVERFLOW +# define WARN_OVERFLOW 15 +#endif + +#ifndef WARN_PACK +# define WARN_PACK 16 +#endif + +#ifndef WARN_PORTABLE +# define WARN_PORTABLE 17 +#endif + +#ifndef WARN_RECURSION +# define WARN_RECURSION 18 +#endif + +#ifndef WARN_REDEFINE +# define WARN_REDEFINE 19 +#endif + +#ifndef WARN_REGEXP +# define WARN_REGEXP 20 +#endif + +#ifndef WARN_SEVERE +# define WARN_SEVERE 21 +#endif + +#ifndef WARN_DEBUGGING +# define WARN_DEBUGGING 22 +#endif + +#ifndef WARN_INPLACE +# define WARN_INPLACE 23 +#endif + +#ifndef WARN_INTERNAL +# define WARN_INTERNAL 24 +#endif + +#ifndef WARN_MALLOC +# define WARN_MALLOC 25 +#endif + +#ifndef WARN_SIGNAL +# define WARN_SIGNAL 26 +#endif + +#ifndef WARN_SUBSTR +# define WARN_SUBSTR 27 +#endif + +#ifndef WARN_SYNTAX +# define WARN_SYNTAX 28 +#endif + +#ifndef WARN_AMBIGUOUS +# define WARN_AMBIGUOUS 29 +#endif + +#ifndef WARN_BAREWORD +# define WARN_BAREWORD 30 +#endif + +#ifndef WARN_DIGIT +# define WARN_DIGIT 31 +#endif + +#ifndef WARN_PARENTHESIS +# define WARN_PARENTHESIS 32 +#endif + +#ifndef WARN_PRECEDENCE +# define WARN_PRECEDENCE 33 +#endif + +#ifndef WARN_PRINTF +# define WARN_PRINTF 34 +#endif + +#ifndef WARN_PROTOTYPE +# define WARN_PROTOTYPE 35 +#endif + +#ifndef WARN_QW +# define WARN_QW 36 +#endif + +#ifndef WARN_RESERVED +# define WARN_RESERVED 37 +#endif + +#ifndef WARN_SEMICOLON +# define WARN_SEMICOLON 38 +#endif + +#ifndef WARN_TAINT +# define WARN_TAINT 39 +#endif + +#ifndef WARN_THREADS +# define WARN_THREADS 40 +#endif + +#ifndef WARN_UNINITIALIZED +# define WARN_UNINITIALIZED 41 +#endif + +#ifndef WARN_UNPACK +# define WARN_UNPACK 42 +#endif + +#ifndef WARN_UNTIE +# define WARN_UNTIE 43 +#endif + +#ifndef WARN_UTF8 +# define WARN_UTF8 44 +#endif + +#ifndef WARN_VOID +# define WARN_VOID 45 +#endif + +#ifndef WARN_ASSERTIONS +# define WARN_ASSERTIONS 46 +#endif +#ifndef packWARN +# define packWARN(a) (a) +#endif + +#ifndef ckWARN +# ifdef G_WARN_ON +# define ckWARN(a) (PL_dowarn & G_WARN_ON) +# else +# define ckWARN(a) PL_dowarn +# endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(warner) +#if defined(NEED_warner) +static void DPPP_(my_warner)(U32 err, const char *pat, ...); +static +#else +extern void DPPP_(my_warner)(U32 err, const char *pat, ...); +#endif + +#define Perl_warner DPPP_(my_warner) + +#if defined(NEED_warner) || defined(NEED_warner_GLOBAL) + +void +DPPP_(my_warner)(U32 err, const char *pat, ...) +{ + SV *sv; + va_list args; + + PERL_UNUSED_ARG(err); + + va_start(args, pat); + sv = vnewSVpvf(pat, &args); + va_end(args); + sv_2mortal(sv); + warn("%s", SvPV_nolen(sv)); +} + +#define warner Perl_warner + +#define Perl_warner_nocontext Perl_warner + +#endif +#endif + +/* concatenating with "" ensures that only literal strings are accepted as argument + * note that STR_WITH_LEN() can't be used as argument to macros or functions that + * under some configurations might be macros + */ +#ifndef STR_WITH_LEN +# define STR_WITH_LEN(s) (s ""), (sizeof(s)-1) +#endif +#ifndef newSVpvs +# define newSVpvs(str) newSVpvn(str "", sizeof(str) - 1) +#endif + +#ifndef newSVpvs_flags +# define newSVpvs_flags(str, flags) newSVpvn_flags(str "", sizeof(str) - 1, flags) +#endif + +#ifndef sv_catpvs +# define sv_catpvs(sv, str) sv_catpvn(sv, str "", sizeof(str) - 1) +#endif + +#ifndef sv_setpvs +# define sv_setpvs(sv, str) sv_setpvn(sv, str "", sizeof(str) - 1) +#endif + +#ifndef hv_fetchs +# define hv_fetchs(hv, key, lval) hv_fetch(hv, key "", sizeof(key) - 1, lval) +#endif + +#ifndef hv_stores +# define hv_stores(hv, key, val) hv_store(hv, key "", sizeof(key) - 1, val, 0) +#endif +#ifndef SvGETMAGIC +# define SvGETMAGIC(x) STMT_START { if (SvGMAGICAL(x)) mg_get(x); } STMT_END +#endif +#ifndef PERL_MAGIC_sv +# define PERL_MAGIC_sv '\0' +#endif + +#ifndef PERL_MAGIC_overload +# define PERL_MAGIC_overload 'A' +#endif + +#ifndef PERL_MAGIC_overload_elem +# define PERL_MAGIC_overload_elem 'a' +#endif + +#ifndef PERL_MAGIC_overload_table +# define PERL_MAGIC_overload_table 'c' +#endif + +#ifndef PERL_MAGIC_bm +# define PERL_MAGIC_bm 'B' +#endif + +#ifndef PERL_MAGIC_regdata +# define PERL_MAGIC_regdata 'D' +#endif + +#ifndef PERL_MAGIC_regdatum +# define PERL_MAGIC_regdatum 'd' +#endif + +#ifndef PERL_MAGIC_env +# define PERL_MAGIC_env 'E' +#endif + +#ifndef PERL_MAGIC_envelem +# define PERL_MAGIC_envelem 'e' +#endif + +#ifndef PERL_MAGIC_fm +# define PERL_MAGIC_fm 'f' +#endif + +#ifndef PERL_MAGIC_regex_global +# define PERL_MAGIC_regex_global 'g' +#endif + +#ifndef PERL_MAGIC_isa +# define PERL_MAGIC_isa 'I' +#endif + +#ifndef PERL_MAGIC_isaelem +# define PERL_MAGIC_isaelem 'i' +#endif + +#ifndef PERL_MAGIC_nkeys +# define PERL_MAGIC_nkeys 'k' +#endif + +#ifndef PERL_MAGIC_dbfile +# define PERL_MAGIC_dbfile 'L' +#endif + +#ifndef PERL_MAGIC_dbline +# define PERL_MAGIC_dbline 'l' +#endif + +#ifndef PERL_MAGIC_mutex +# define PERL_MAGIC_mutex 'm' +#endif + +#ifndef PERL_MAGIC_shared +# define PERL_MAGIC_shared 'N' +#endif + +#ifndef PERL_MAGIC_shared_scalar +# define PERL_MAGIC_shared_scalar 'n' +#endif + +#ifndef PERL_MAGIC_collxfrm +# define PERL_MAGIC_collxfrm 'o' +#endif + +#ifndef PERL_MAGIC_tied +# define PERL_MAGIC_tied 'P' +#endif + +#ifndef PERL_MAGIC_tiedelem +# define PERL_MAGIC_tiedelem 'p' +#endif + +#ifndef PERL_MAGIC_tiedscalar +# define PERL_MAGIC_tiedscalar 'q' +#endif + +#ifndef PERL_MAGIC_qr +# define PERL_MAGIC_qr 'r' +#endif + +#ifndef PERL_MAGIC_sig +# define PERL_MAGIC_sig 'S' +#endif + +#ifndef PERL_MAGIC_sigelem +# define PERL_MAGIC_sigelem 's' +#endif + +#ifndef PERL_MAGIC_taint +# define PERL_MAGIC_taint 't' +#endif + +#ifndef PERL_MAGIC_uvar +# define PERL_MAGIC_uvar 'U' +#endif + +#ifndef PERL_MAGIC_uvar_elem +# define PERL_MAGIC_uvar_elem 'u' +#endif + +#ifndef PERL_MAGIC_vstring +# define PERL_MAGIC_vstring 'V' +#endif + +#ifndef PERL_MAGIC_vec +# define PERL_MAGIC_vec 'v' +#endif + +#ifndef PERL_MAGIC_utf8 +# define PERL_MAGIC_utf8 'w' +#endif + +#ifndef PERL_MAGIC_substr +# define PERL_MAGIC_substr 'x' +#endif + +#ifndef PERL_MAGIC_defelem +# define PERL_MAGIC_defelem 'y' +#endif + +#ifndef PERL_MAGIC_glob +# define PERL_MAGIC_glob '*' +#endif + +#ifndef PERL_MAGIC_arylen +# define PERL_MAGIC_arylen '#' +#endif + +#ifndef PERL_MAGIC_pos +# define PERL_MAGIC_pos '.' +#endif + +#ifndef PERL_MAGIC_backref +# define PERL_MAGIC_backref '<' +#endif + +#ifndef PERL_MAGIC_ext +# define PERL_MAGIC_ext '~' +#endif + +/* That's the best we can do... */ +#ifndef sv_catpvn_nomg +# define sv_catpvn_nomg sv_catpvn +#endif + +#ifndef sv_catsv_nomg +# define sv_catsv_nomg sv_catsv +#endif + +#ifndef sv_setsv_nomg +# define sv_setsv_nomg sv_setsv +#endif + +#ifndef sv_pvn_nomg +# define sv_pvn_nomg sv_pvn +#endif + +#ifndef SvIV_nomg +# define SvIV_nomg SvIV +#endif + +#ifndef SvUV_nomg +# define SvUV_nomg SvUV +#endif + +#ifndef sv_catpv_mg +# define sv_catpv_mg(sv, ptr) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_catpv(TeMpSv,ptr); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_catpvn_mg +# define sv_catpvn_mg(sv, ptr, len) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_catpvn(TeMpSv,ptr,len); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_catsv_mg +# define sv_catsv_mg(dsv, ssv) \ + STMT_START { \ + SV *TeMpSv = dsv; \ + sv_catsv(TeMpSv,ssv); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setiv_mg +# define sv_setiv_mg(sv, i) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setiv(TeMpSv,i); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setnv_mg +# define sv_setnv_mg(sv, num) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setnv(TeMpSv,num); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setpv_mg +# define sv_setpv_mg(sv, ptr) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setpv(TeMpSv,ptr); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setpvn_mg +# define sv_setpvn_mg(sv, ptr, len) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setpvn(TeMpSv,ptr,len); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setsv_mg +# define sv_setsv_mg(dsv, ssv) \ + STMT_START { \ + SV *TeMpSv = dsv; \ + sv_setsv(TeMpSv,ssv); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setuv_mg +# define sv_setuv_mg(sv, i) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setuv(TeMpSv,i); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_usepvn_mg +# define sv_usepvn_mg(sv, ptr, len) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_usepvn(TeMpSv,ptr,len); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif +#ifndef SvVSTRING_mg +# define SvVSTRING_mg(sv) (SvMAGICAL(sv) ? mg_find(sv, PERL_MAGIC_vstring) : NULL) +#endif + +/* Hint: sv_magic_portable + * This is a compatibility function that is only available with + * Devel::PPPort. It is NOT in the perl core. + * Its purpose is to mimic the 5.8.0 behaviour of sv_magic() when + * it is being passed a name pointer with namlen == 0. In that + * case, perl 5.8.0 and later store the pointer, not a copy of it. + * The compatibility can be provided back to perl 5.004. With + * earlier versions, the code will not compile. + */ + +#if (PERL_BCDVERSION < 0x5004000) + + /* code that uses sv_magic_portable will not compile */ + +#elif (PERL_BCDVERSION < 0x5008000) + +# define sv_magic_portable(sv, obj, how, name, namlen) \ + STMT_START { \ + SV *SvMp_sv = (sv); \ + char *SvMp_name = (char *) (name); \ + I32 SvMp_namlen = (namlen); \ + if (SvMp_name && SvMp_namlen == 0) \ + { \ + MAGIC *mg; \ + sv_magic(SvMp_sv, obj, how, 0, 0); \ + mg = SvMAGIC(SvMp_sv); \ + mg->mg_len = -42; /* XXX: this is the tricky part */ \ + mg->mg_ptr = SvMp_name; \ + } \ + else \ + { \ + sv_magic(SvMp_sv, obj, how, SvMp_name, SvMp_namlen); \ + } \ + } STMT_END + +#else + +# define sv_magic_portable(a, b, c, d, e) sv_magic(a, b, c, d, e) + +#endif + +#ifdef USE_ITHREADS +#ifndef CopFILE +# define CopFILE(c) ((c)->cop_file) +#endif + +#ifndef CopFILEGV +# define CopFILEGV(c) (CopFILE(c) ? gv_fetchfile(CopFILE(c)) : Nullgv) +#endif + +#ifndef CopFILE_set +# define CopFILE_set(c,pv) ((c)->cop_file = savepv(pv)) +#endif + +#ifndef CopFILESV +# define CopFILESV(c) (CopFILE(c) ? GvSV(gv_fetchfile(CopFILE(c))) : Nullsv) +#endif + +#ifndef CopFILEAV +# define CopFILEAV(c) (CopFILE(c) ? GvAV(gv_fetchfile(CopFILE(c))) : Nullav) +#endif + +#ifndef CopSTASHPV +# define CopSTASHPV(c) ((c)->cop_stashpv) +#endif + +#ifndef CopSTASHPV_set +# define CopSTASHPV_set(c,pv) ((c)->cop_stashpv = ((pv) ? savepv(pv) : Nullch)) +#endif + +#ifndef CopSTASH +# define CopSTASH(c) (CopSTASHPV(c) ? gv_stashpv(CopSTASHPV(c),GV_ADD) : Nullhv) +#endif + +#ifndef CopSTASH_set +# define CopSTASH_set(c,hv) CopSTASHPV_set(c, (hv) ? HvNAME(hv) : Nullch) +#endif + +#ifndef CopSTASH_eq +# define CopSTASH_eq(c,hv) ((hv) && (CopSTASHPV(c) == HvNAME(hv) \ + || (CopSTASHPV(c) && HvNAME(hv) \ + && strEQ(CopSTASHPV(c), HvNAME(hv))))) +#endif + +#else +#ifndef CopFILEGV +# define CopFILEGV(c) ((c)->cop_filegv) +#endif + +#ifndef CopFILEGV_set +# define CopFILEGV_set(c,gv) ((c)->cop_filegv = (GV*)SvREFCNT_inc(gv)) +#endif + +#ifndef CopFILE_set +# define CopFILE_set(c,pv) CopFILEGV_set((c), gv_fetchfile(pv)) +#endif + +#ifndef CopFILESV +# define CopFILESV(c) (CopFILEGV(c) ? GvSV(CopFILEGV(c)) : Nullsv) +#endif + +#ifndef CopFILEAV +# define CopFILEAV(c) (CopFILEGV(c) ? GvAV(CopFILEGV(c)) : Nullav) +#endif + +#ifndef CopFILE +# define CopFILE(c) (CopFILESV(c) ? SvPVX(CopFILESV(c)) : Nullch) +#endif + +#ifndef CopSTASH +# define CopSTASH(c) ((c)->cop_stash) +#endif + +#ifndef CopSTASH_set +# define CopSTASH_set(c,hv) ((c)->cop_stash = (hv)) +#endif + +#ifndef CopSTASHPV +# define CopSTASHPV(c) (CopSTASH(c) ? HvNAME(CopSTASH(c)) : Nullch) +#endif + +#ifndef CopSTASHPV_set +# define CopSTASHPV_set(c,pv) CopSTASH_set((c), gv_stashpv(pv,GV_ADD)) +#endif + +#ifndef CopSTASH_eq +# define CopSTASH_eq(c,hv) (CopSTASH(c) == (hv)) +#endif + +#endif /* USE_ITHREADS */ +#ifndef IN_PERL_COMPILETIME +# define IN_PERL_COMPILETIME (PL_curcop == &PL_compiling) +#endif + +#ifndef IN_LOCALE_RUNTIME +# define IN_LOCALE_RUNTIME (PL_curcop->op_private & HINT_LOCALE) +#endif + +#ifndef IN_LOCALE_COMPILETIME +# define IN_LOCALE_COMPILETIME (PL_hints & HINT_LOCALE) +#endif + +#ifndef IN_LOCALE +# define IN_LOCALE (IN_PERL_COMPILETIME ? IN_LOCALE_COMPILETIME : IN_LOCALE_RUNTIME) +#endif +#ifndef IS_NUMBER_IN_UV +# define IS_NUMBER_IN_UV 0x01 +#endif + +#ifndef IS_NUMBER_GREATER_THAN_UV_MAX +# define IS_NUMBER_GREATER_THAN_UV_MAX 0x02 +#endif + +#ifndef IS_NUMBER_NOT_INT +# define IS_NUMBER_NOT_INT 0x04 +#endif + +#ifndef IS_NUMBER_NEG +# define IS_NUMBER_NEG 0x08 +#endif + +#ifndef IS_NUMBER_INFINITY +# define IS_NUMBER_INFINITY 0x10 +#endif + +#ifndef IS_NUMBER_NAN +# define IS_NUMBER_NAN 0x20 +#endif +#ifndef GROK_NUMERIC_RADIX +# define GROK_NUMERIC_RADIX(sp, send) grok_numeric_radix(sp, send) +#endif +#ifndef PERL_SCAN_GREATER_THAN_UV_MAX +# define PERL_SCAN_GREATER_THAN_UV_MAX 0x02 +#endif + +#ifndef PERL_SCAN_SILENT_ILLDIGIT +# define PERL_SCAN_SILENT_ILLDIGIT 0x04 +#endif + +#ifndef PERL_SCAN_ALLOW_UNDERSCORES +# define PERL_SCAN_ALLOW_UNDERSCORES 0x01 +#endif + +#ifndef PERL_SCAN_DISALLOW_PREFIX +# define PERL_SCAN_DISALLOW_PREFIX 0x02 +#endif + +#ifndef grok_numeric_radix +#if defined(NEED_grok_numeric_radix) +static bool DPPP_(my_grok_numeric_radix)(pTHX_ const char ** sp, const char * send); +static +#else +extern bool DPPP_(my_grok_numeric_radix)(pTHX_ const char ** sp, const char * send); +#endif + +#ifdef grok_numeric_radix +# undef grok_numeric_radix +#endif +#define grok_numeric_radix(a,b) DPPP_(my_grok_numeric_radix)(aTHX_ a,b) +#define Perl_grok_numeric_radix DPPP_(my_grok_numeric_radix) + +#if defined(NEED_grok_numeric_radix) || defined(NEED_grok_numeric_radix_GLOBAL) +bool +DPPP_(my_grok_numeric_radix)(pTHX_ const char **sp, const char *send) +{ +#ifdef USE_LOCALE_NUMERIC +#ifdef PL_numeric_radix_sv + if (PL_numeric_radix_sv && IN_LOCALE) { + STRLEN len; + char* radix = SvPV(PL_numeric_radix_sv, len); + if (*sp + len <= send && memEQ(*sp, radix, len)) { + *sp += len; + return TRUE; + } + } +#else + /* older perls don't have PL_numeric_radix_sv so the radix + * must manually be requested from locale.h + */ +#include + dTHR; /* needed for older threaded perls */ + struct lconv *lc = localeconv(); + char *radix = lc->decimal_point; + if (radix && IN_LOCALE) { + STRLEN len = strlen(radix); + if (*sp + len <= send && memEQ(*sp, radix, len)) { + *sp += len; + return TRUE; + } + } +#endif +#endif /* USE_LOCALE_NUMERIC */ + /* always try "." if numeric radix didn't match because + * we may have data from different locales mixed */ + if (*sp < send && **sp == '.') { + ++*sp; + return TRUE; + } + return FALSE; +} +#endif +#endif + +#ifndef grok_number +#if defined(NEED_grok_number) +static int DPPP_(my_grok_number)(pTHX_ const char * pv, STRLEN len, UV * valuep); +static +#else +extern int DPPP_(my_grok_number)(pTHX_ const char * pv, STRLEN len, UV * valuep); +#endif + +#ifdef grok_number +# undef grok_number +#endif +#define grok_number(a,b,c) DPPP_(my_grok_number)(aTHX_ a,b,c) +#define Perl_grok_number DPPP_(my_grok_number) + +#if defined(NEED_grok_number) || defined(NEED_grok_number_GLOBAL) +int +DPPP_(my_grok_number)(pTHX_ const char *pv, STRLEN len, UV *valuep) +{ + const char *s = pv; + const char *send = pv + len; + const UV max_div_10 = UV_MAX / 10; + const char max_mod_10 = UV_MAX % 10; + int numtype = 0; + int sawinf = 0; + int sawnan = 0; + + while (s < send && isSPACE(*s)) + s++; + if (s == send) { + return 0; + } else if (*s == '-') { + s++; + numtype = IS_NUMBER_NEG; + } + else if (*s == '+') + s++; + + if (s == send) + return 0; + + /* next must be digit or the radix separator or beginning of infinity */ + if (isDIGIT(*s)) { + /* UVs are at least 32 bits, so the first 9 decimal digits cannot + overflow. */ + UV value = *s - '0'; + /* This construction seems to be more optimiser friendly. + (without it gcc does the isDIGIT test and the *s - '0' separately) + With it gcc on arm is managing 6 instructions (6 cycles) per digit. + In theory the optimiser could deduce how far to unroll the loop + before checking for overflow. */ + if (++s < send) { + int digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + /* Now got 9 digits, so need to check + each time for overflow. */ + digit = *s - '0'; + while (digit >= 0 && digit <= 9 + && (value < max_div_10 + || (value == max_div_10 + && digit <= max_mod_10))) { + value = value * 10 + digit; + if (++s < send) + digit = *s - '0'; + else + break; + } + if (digit >= 0 && digit <= 9 + && (s < send)) { + /* value overflowed. + skip the remaining digits, don't + worry about setting *valuep. */ + do { + s++; + } while (s < send && isDIGIT(*s)); + numtype |= + IS_NUMBER_GREATER_THAN_UV_MAX; + goto skip_value; + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + numtype |= IS_NUMBER_IN_UV; + if (valuep) + *valuep = value; + + skip_value: + if (GROK_NUMERIC_RADIX(&s, send)) { + numtype |= IS_NUMBER_NOT_INT; + while (s < send && isDIGIT(*s)) /* optional digits after the radix */ + s++; + } + } + else if (GROK_NUMERIC_RADIX(&s, send)) { + numtype |= IS_NUMBER_NOT_INT | IS_NUMBER_IN_UV; /* valuep assigned below */ + /* no digits before the radix means we need digits after it */ + if (s < send && isDIGIT(*s)) { + do { + s++; + } while (s < send && isDIGIT(*s)); + if (valuep) { + /* integer approximation is valid - it's 0. */ + *valuep = 0; + } + } + else + return 0; + } else if (*s == 'I' || *s == 'i') { + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; if (s == send || (*s != 'F' && *s != 'f')) return 0; + s++; if (s < send && (*s == 'I' || *s == 'i')) { + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; if (s == send || (*s != 'I' && *s != 'i')) return 0; + s++; if (s == send || (*s != 'T' && *s != 't')) return 0; + s++; if (s == send || (*s != 'Y' && *s != 'y')) return 0; + s++; + } + sawinf = 1; + } else if (*s == 'N' || *s == 'n') { + /* XXX TODO: There are signaling NaNs and quiet NaNs. */ + s++; if (s == send || (*s != 'A' && *s != 'a')) return 0; + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; + sawnan = 1; + } else + return 0; + + if (sawinf) { + numtype &= IS_NUMBER_NEG; /* Keep track of sign */ + numtype |= IS_NUMBER_INFINITY | IS_NUMBER_NOT_INT; + } else if (sawnan) { + numtype &= IS_NUMBER_NEG; /* Keep track of sign */ + numtype |= IS_NUMBER_NAN | IS_NUMBER_NOT_INT; + } else if (s < send) { + /* we can have an optional exponent part */ + if (*s == 'e' || *s == 'E') { + /* The only flag we keep is sign. Blow away any "it's UV" */ + numtype &= IS_NUMBER_NEG; + numtype |= IS_NUMBER_NOT_INT; + s++; + if (s < send && (*s == '-' || *s == '+')) + s++; + if (s < send && isDIGIT(*s)) { + do { + s++; + } while (s < send && isDIGIT(*s)); + } + else + return 0; + } + } + while (s < send && isSPACE(*s)) + s++; + if (s >= send) + return numtype; + if (len == 10 && memEQ(pv, "0 but true", 10)) { + if (valuep) + *valuep = 0; + return IS_NUMBER_IN_UV; + } + return 0; +} +#endif +#endif + +/* + * The grok_* routines have been modified to use warn() instead of + * Perl_warner(). Also, 'hexdigit' was the former name of PL_hexdigit, + * which is why the stack variable has been renamed to 'xdigit'. + */ + +#ifndef grok_bin +#if defined(NEED_grok_bin) +static UV DPPP_(my_grok_bin)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +static +#else +extern UV DPPP_(my_grok_bin)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +#endif + +#ifdef grok_bin +# undef grok_bin +#endif +#define grok_bin(a,b,c,d) DPPP_(my_grok_bin)(aTHX_ a,b,c,d) +#define Perl_grok_bin DPPP_(my_grok_bin) + +#if defined(NEED_grok_bin) || defined(NEED_grok_bin_GLOBAL) +UV +DPPP_(my_grok_bin)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) +{ + const char *s = start; + STRLEN len = *len_p; + UV value = 0; + NV value_nv = 0; + + const UV max_div_2 = UV_MAX / 2; + bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; + bool overflowed = FALSE; + + if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) { + /* strip off leading b or 0b. + for compatibility silently suffer "b" and "0b" as valid binary + numbers. */ + if (len >= 1) { + if (s[0] == 'b') { + s++; + len--; + } + else if (len >= 2 && s[0] == '0' && s[1] == 'b') { + s+=2; + len-=2; + } + } + } + + for (; len-- && *s; s++) { + char bit = *s; + if (bit == '0' || bit == '1') { + /* Write it in this wonky order with a goto to attempt to get the + compiler to make the common case integer-only loop pretty tight. + With gcc seems to be much straighter code than old scan_bin. */ + redo: + if (!overflowed) { + if (value <= max_div_2) { + value = (value << 1) | (bit - '0'); + continue; + } + /* Bah. We're just overflowed. */ + warn("Integer overflow in binary number"); + overflowed = TRUE; + value_nv = (NV) value; + } + value_nv *= 2.0; + /* If an NV has not enough bits in its mantissa to + * represent a UV this summing of small low-order numbers + * is a waste of time (because the NV cannot preserve + * the low-order bits anyway): we could just remember when + * did we overflow and in the end just multiply value_nv by the + * right amount. */ + value_nv += (NV)(bit - '0'); + continue; + } + if (bit == '_' && len && allow_underscores && (bit = s[1]) + && (bit == '0' || bit == '1')) + { + --len; + ++s; + goto redo; + } + if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) + warn("Illegal binary digit '%c' ignored", *s); + break; + } + + if ( ( overflowed && value_nv > 4294967295.0) +#if UVSIZE > 4 + || (!overflowed && value > 0xffffffff ) +#endif + ) { + warn("Binary number > 0b11111111111111111111111111111111 non-portable"); + } + *len_p = s - start; + if (!overflowed) { + *flags = 0; + return value; + } + *flags = PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) + *result = value_nv; + return UV_MAX; +} +#endif +#endif + +#ifndef grok_hex +#if defined(NEED_grok_hex) +static UV DPPP_(my_grok_hex)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +static +#else +extern UV DPPP_(my_grok_hex)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +#endif + +#ifdef grok_hex +# undef grok_hex +#endif +#define grok_hex(a,b,c,d) DPPP_(my_grok_hex)(aTHX_ a,b,c,d) +#define Perl_grok_hex DPPP_(my_grok_hex) + +#if defined(NEED_grok_hex) || defined(NEED_grok_hex_GLOBAL) +UV +DPPP_(my_grok_hex)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) +{ + const char *s = start; + STRLEN len = *len_p; + UV value = 0; + NV value_nv = 0; + + const UV max_div_16 = UV_MAX / 16; + bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; + bool overflowed = FALSE; + const char *xdigit; + + if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) { + /* strip off leading x or 0x. + for compatibility silently suffer "x" and "0x" as valid hex numbers. + */ + if (len >= 1) { + if (s[0] == 'x') { + s++; + len--; + } + else if (len >= 2 && s[0] == '0' && s[1] == 'x') { + s+=2; + len-=2; + } + } + } + + for (; len-- && *s; s++) { + xdigit = strchr((char *) PL_hexdigit, *s); + if (xdigit) { + /* Write it in this wonky order with a goto to attempt to get the + compiler to make the common case integer-only loop pretty tight. + With gcc seems to be much straighter code than old scan_hex. */ + redo: + if (!overflowed) { + if (value <= max_div_16) { + value = (value << 4) | ((xdigit - PL_hexdigit) & 15); + continue; + } + warn("Integer overflow in hexadecimal number"); + overflowed = TRUE; + value_nv = (NV) value; + } + value_nv *= 16.0; + /* If an NV has not enough bits in its mantissa to + * represent a UV this summing of small low-order numbers + * is a waste of time (because the NV cannot preserve + * the low-order bits anyway): we could just remember when + * did we overflow and in the end just multiply value_nv by the + * right amount of 16-tuples. */ + value_nv += (NV)((xdigit - PL_hexdigit) & 15); + continue; + } + if (*s == '_' && len && allow_underscores && s[1] + && (xdigit = strchr((char *) PL_hexdigit, s[1]))) + { + --len; + ++s; + goto redo; + } + if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) + warn("Illegal hexadecimal digit '%c' ignored", *s); + break; + } + + if ( ( overflowed && value_nv > 4294967295.0) +#if UVSIZE > 4 + || (!overflowed && value > 0xffffffff ) +#endif + ) { + warn("Hexadecimal number > 0xffffffff non-portable"); + } + *len_p = s - start; + if (!overflowed) { + *flags = 0; + return value; + } + *flags = PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) + *result = value_nv; + return UV_MAX; +} +#endif +#endif + +#ifndef grok_oct +#if defined(NEED_grok_oct) +static UV DPPP_(my_grok_oct)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +static +#else +extern UV DPPP_(my_grok_oct)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +#endif + +#ifdef grok_oct +# undef grok_oct +#endif +#define grok_oct(a,b,c,d) DPPP_(my_grok_oct)(aTHX_ a,b,c,d) +#define Perl_grok_oct DPPP_(my_grok_oct) + +#if defined(NEED_grok_oct) || defined(NEED_grok_oct_GLOBAL) +UV +DPPP_(my_grok_oct)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) +{ + const char *s = start; + STRLEN len = *len_p; + UV value = 0; + NV value_nv = 0; + + const UV max_div_8 = UV_MAX / 8; + bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; + bool overflowed = FALSE; + + for (; len-- && *s; s++) { + /* gcc 2.95 optimiser not smart enough to figure that this subtraction + out front allows slicker code. */ + int digit = *s - '0'; + if (digit >= 0 && digit <= 7) { + /* Write it in this wonky order with a goto to attempt to get the + compiler to make the common case integer-only loop pretty tight. + */ + redo: + if (!overflowed) { + if (value <= max_div_8) { + value = (value << 3) | digit; + continue; + } + /* Bah. We're just overflowed. */ + warn("Integer overflow in octal number"); + overflowed = TRUE; + value_nv = (NV) value; + } + value_nv *= 8.0; + /* If an NV has not enough bits in its mantissa to + * represent a UV this summing of small low-order numbers + * is a waste of time (because the NV cannot preserve + * the low-order bits anyway): we could just remember when + * did we overflow and in the end just multiply value_nv by the + * right amount of 8-tuples. */ + value_nv += (NV)digit; + continue; + } + if (digit == ('_' - '0') && len && allow_underscores + && (digit = s[1] - '0') && (digit >= 0 && digit <= 7)) + { + --len; + ++s; + goto redo; + } + /* Allow \octal to work the DWIM way (that is, stop scanning + * as soon as non-octal characters are seen, complain only iff + * someone seems to want to use the digits eight and nine). */ + if (digit == 8 || digit == 9) { + if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) + warn("Illegal octal digit '%c' ignored", *s); + } + break; + } + + if ( ( overflowed && value_nv > 4294967295.0) +#if UVSIZE > 4 + || (!overflowed && value > 0xffffffff ) +#endif + ) { + warn("Octal number > 037777777777 non-portable"); + } + *len_p = s - start; + if (!overflowed) { + *flags = 0; + return value; + } + *flags = PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) + *result = value_nv; + return UV_MAX; +} +#endif +#endif + +#if !defined(my_snprintf) +#if defined(NEED_my_snprintf) +static int DPPP_(my_my_snprintf)(char * buffer, const Size_t len, const char * format, ...); +static +#else +extern int DPPP_(my_my_snprintf)(char * buffer, const Size_t len, const char * format, ...); +#endif + +#define my_snprintf DPPP_(my_my_snprintf) +#define Perl_my_snprintf DPPP_(my_my_snprintf) + +#if defined(NEED_my_snprintf) || defined(NEED_my_snprintf_GLOBAL) + +int +DPPP_(my_my_snprintf)(char *buffer, const Size_t len, const char *format, ...) +{ + dTHX; + int retval; + va_list ap; + va_start(ap, format); +#ifdef HAS_VSNPRINTF + retval = vsnprintf(buffer, len, format, ap); +#else + retval = vsprintf(buffer, format, ap); +#endif + va_end(ap); + if (retval < 0 || (len > 0 && (Size_t)retval >= len)) + Perl_croak(aTHX_ "panic: my_snprintf buffer overflow"); + return retval; +} + +#endif +#endif + +#if !defined(my_sprintf) +#if defined(NEED_my_sprintf) +static int DPPP_(my_my_sprintf)(char * buffer, const char * pat, ...); +static +#else +extern int DPPP_(my_my_sprintf)(char * buffer, const char * pat, ...); +#endif + +#define my_sprintf DPPP_(my_my_sprintf) +#define Perl_my_sprintf DPPP_(my_my_sprintf) + +#if defined(NEED_my_sprintf) || defined(NEED_my_sprintf_GLOBAL) + +int +DPPP_(my_my_sprintf)(char *buffer, const char* pat, ...) +{ + va_list args; + va_start(args, pat); + vsprintf(buffer, pat, args); + va_end(args); + return strlen(buffer); +} + +#endif +#endif + +#ifdef NO_XSLOCKS +# ifdef dJMPENV +# define dXCPT dJMPENV; int rEtV = 0 +# define XCPT_TRY_START JMPENV_PUSH(rEtV); if (rEtV == 0) +# define XCPT_TRY_END JMPENV_POP; +# define XCPT_CATCH if (rEtV != 0) +# define XCPT_RETHROW JMPENV_JUMP(rEtV) +# else +# define dXCPT Sigjmp_buf oldTOP; int rEtV = 0 +# define XCPT_TRY_START Copy(top_env, oldTOP, 1, Sigjmp_buf); rEtV = Sigsetjmp(top_env, 1); if (rEtV == 0) +# define XCPT_TRY_END Copy(oldTOP, top_env, 1, Sigjmp_buf); +# define XCPT_CATCH if (rEtV != 0) +# define XCPT_RETHROW Siglongjmp(top_env, rEtV) +# endif +#endif + +#if !defined(my_strlcat) +#if defined(NEED_my_strlcat) +static Size_t DPPP_(my_my_strlcat)(char * dst, const char * src, Size_t size); +static +#else +extern Size_t DPPP_(my_my_strlcat)(char * dst, const char * src, Size_t size); +#endif + +#define my_strlcat DPPP_(my_my_strlcat) +#define Perl_my_strlcat DPPP_(my_my_strlcat) + +#if defined(NEED_my_strlcat) || defined(NEED_my_strlcat_GLOBAL) + +Size_t +DPPP_(my_my_strlcat)(char *dst, const char *src, Size_t size) +{ + Size_t used, length, copy; + + used = strlen(dst); + length = strlen(src); + if (size > 0 && used < size - 1) { + copy = (length >= size - used) ? size - used - 1 : length; + memcpy(dst + used, src, copy); + dst[used + copy] = '\0'; + } + return used + length; +} +#endif +#endif + +#if !defined(my_strlcpy) +#if defined(NEED_my_strlcpy) +static Size_t DPPP_(my_my_strlcpy)(char * dst, const char * src, Size_t size); +static +#else +extern Size_t DPPP_(my_my_strlcpy)(char * dst, const char * src, Size_t size); +#endif + +#define my_strlcpy DPPP_(my_my_strlcpy) +#define Perl_my_strlcpy DPPP_(my_my_strlcpy) + +#if defined(NEED_my_strlcpy) || defined(NEED_my_strlcpy_GLOBAL) + +Size_t +DPPP_(my_my_strlcpy)(char *dst, const char *src, Size_t size) +{ + Size_t length, copy; + + length = strlen(src); + if (size > 0) { + copy = (length >= size) ? size - 1 : length; + memcpy(dst, src, copy); + dst[copy] = '\0'; + } + return length; +} + +#endif +#endif +#ifndef PERL_PV_ESCAPE_QUOTE +# define PERL_PV_ESCAPE_QUOTE 0x0001 +#endif + +#ifndef PERL_PV_PRETTY_QUOTE +# define PERL_PV_PRETTY_QUOTE PERL_PV_ESCAPE_QUOTE +#endif + +#ifndef PERL_PV_PRETTY_ELLIPSES +# define PERL_PV_PRETTY_ELLIPSES 0x0002 +#endif + +#ifndef PERL_PV_PRETTY_LTGT +# define PERL_PV_PRETTY_LTGT 0x0004 +#endif + +#ifndef PERL_PV_ESCAPE_FIRSTCHAR +# define PERL_PV_ESCAPE_FIRSTCHAR 0x0008 +#endif + +#ifndef PERL_PV_ESCAPE_UNI +# define PERL_PV_ESCAPE_UNI 0x0100 +#endif + +#ifndef PERL_PV_ESCAPE_UNI_DETECT +# define PERL_PV_ESCAPE_UNI_DETECT 0x0200 +#endif + +#ifndef PERL_PV_ESCAPE_ALL +# define PERL_PV_ESCAPE_ALL 0x1000 +#endif + +#ifndef PERL_PV_ESCAPE_NOBACKSLASH +# define PERL_PV_ESCAPE_NOBACKSLASH 0x2000 +#endif + +#ifndef PERL_PV_ESCAPE_NOCLEAR +# define PERL_PV_ESCAPE_NOCLEAR 0x4000 +#endif + +#ifndef PERL_PV_ESCAPE_RE +# define PERL_PV_ESCAPE_RE 0x8000 +#endif + +#ifndef PERL_PV_PRETTY_NOCLEAR +# define PERL_PV_PRETTY_NOCLEAR PERL_PV_ESCAPE_NOCLEAR +#endif +#ifndef PERL_PV_PRETTY_DUMP +# define PERL_PV_PRETTY_DUMP PERL_PV_PRETTY_ELLIPSES|PERL_PV_PRETTY_QUOTE +#endif + +#ifndef PERL_PV_PRETTY_REGPROP +# define PERL_PV_PRETTY_REGPROP PERL_PV_PRETTY_ELLIPSES|PERL_PV_PRETTY_LTGT|PERL_PV_ESCAPE_RE +#endif + +/* Hint: pv_escape + * Note that unicode functionality is only backported to + * those perl versions that support it. For older perl + * versions, the implementation will fall back to bytes. + */ + +#ifndef pv_escape +#if defined(NEED_pv_escape) +static char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags); +static +#else +extern char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags); +#endif + +#ifdef pv_escape +# undef pv_escape +#endif +#define pv_escape(a,b,c,d,e,f) DPPP_(my_pv_escape)(aTHX_ a,b,c,d,e,f) +#define Perl_pv_escape DPPP_(my_pv_escape) + +#if defined(NEED_pv_escape) || defined(NEED_pv_escape_GLOBAL) + +char * +DPPP_(my_pv_escape)(pTHX_ SV *dsv, char const * const str, + const STRLEN count, const STRLEN max, + STRLEN * const escaped, const U32 flags) +{ + const char esc = flags & PERL_PV_ESCAPE_RE ? '%' : '\\'; + const char dq = flags & PERL_PV_ESCAPE_QUOTE ? '"' : esc; + char octbuf[32] = "%123456789ABCDF"; + STRLEN wrote = 0; + STRLEN chsize = 0; + STRLEN readsize = 1; +#if defined(is_utf8_string) && defined(utf8_to_uvchr) + bool isuni = flags & PERL_PV_ESCAPE_UNI ? 1 : 0; +#endif + const char *pv = str; + const char * const end = pv + count; + octbuf[0] = esc; + + if (!(flags & PERL_PV_ESCAPE_NOCLEAR)) + sv_setpvs(dsv, ""); + +#if defined(is_utf8_string) && defined(utf8_to_uvchr) + if ((flags & PERL_PV_ESCAPE_UNI_DETECT) && is_utf8_string((U8*)pv, count)) + isuni = 1; +#endif + + for (; pv < end && (!max || wrote < max) ; pv += readsize) { + const UV u = +#if defined(is_utf8_string) && defined(utf8_to_uvchr) + isuni ? utf8_to_uvchr((U8*)pv, &readsize) : +#endif + (U8)*pv; + const U8 c = (U8)u & 0xFF; + + if (u > 255 || (flags & PERL_PV_ESCAPE_ALL)) { + if (flags & PERL_PV_ESCAPE_FIRSTCHAR) + chsize = my_snprintf(octbuf, sizeof octbuf, + "%"UVxf, u); + else + chsize = my_snprintf(octbuf, sizeof octbuf, + "%cx{%"UVxf"}", esc, u); + } else if (flags & PERL_PV_ESCAPE_NOBACKSLASH) { + chsize = 1; + } else { + if (c == dq || c == esc || !isPRINT(c)) { + chsize = 2; + switch (c) { + case '\\' : /* fallthrough */ + case '%' : if (c == esc) + octbuf[1] = esc; + else + chsize = 1; + break; + case '\v' : octbuf[1] = 'v'; break; + case '\t' : octbuf[1] = 't'; break; + case '\r' : octbuf[1] = 'r'; break; + case '\n' : octbuf[1] = 'n'; break; + case '\f' : octbuf[1] = 'f'; break; + case '"' : if (dq == '"') + octbuf[1] = '"'; + else + chsize = 1; + break; + default: chsize = my_snprintf(octbuf, sizeof octbuf, + pv < end && isDIGIT((U8)*(pv+readsize)) + ? "%c%03o" : "%c%o", esc, c); + } + } else { + chsize = 1; + } + } + if (max && wrote + chsize > max) { + break; + } else if (chsize > 1) { + sv_catpvn(dsv, octbuf, chsize); + wrote += chsize; + } else { + char tmp[2]; + my_snprintf(tmp, sizeof tmp, "%c", c); + sv_catpvn(dsv, tmp, 1); + wrote++; + } + if (flags & PERL_PV_ESCAPE_FIRSTCHAR) + break; + } + if (escaped != NULL) + *escaped= pv - str; + return SvPVX(dsv); +} + +#endif +#endif + +#ifndef pv_pretty +#if defined(NEED_pv_pretty) +static char * DPPP_(my_pv_pretty)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, char const * const start_color, char const * const end_color, const U32 flags); +static +#else +extern char * DPPP_(my_pv_pretty)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, char const * const start_color, char const * const end_color, const U32 flags); +#endif + +#ifdef pv_pretty +# undef pv_pretty +#endif +#define pv_pretty(a,b,c,d,e,f,g) DPPP_(my_pv_pretty)(aTHX_ a,b,c,d,e,f,g) +#define Perl_pv_pretty DPPP_(my_pv_pretty) + +#if defined(NEED_pv_pretty) || defined(NEED_pv_pretty_GLOBAL) + +char * +DPPP_(my_pv_pretty)(pTHX_ SV *dsv, char const * const str, const STRLEN count, + const STRLEN max, char const * const start_color, char const * const end_color, + const U32 flags) +{ + const U8 dq = (flags & PERL_PV_PRETTY_QUOTE) ? '"' : '%'; + STRLEN escaped; + + if (!(flags & PERL_PV_PRETTY_NOCLEAR)) + sv_setpvs(dsv, ""); + + if (dq == '"') + sv_catpvs(dsv, "\""); + else if (flags & PERL_PV_PRETTY_LTGT) + sv_catpvs(dsv, "<"); + + if (start_color != NULL) + sv_catpv(dsv, D_PPP_CONSTPV_ARG(start_color)); + + pv_escape(dsv, str, count, max, &escaped, flags | PERL_PV_ESCAPE_NOCLEAR); + + if (end_color != NULL) + sv_catpv(dsv, D_PPP_CONSTPV_ARG(end_color)); + + if (dq == '"') + sv_catpvs(dsv, "\""); + else if (flags & PERL_PV_PRETTY_LTGT) + sv_catpvs(dsv, ">"); + + if ((flags & PERL_PV_PRETTY_ELLIPSES) && escaped < count) + sv_catpvs(dsv, "..."); + + return SvPVX(dsv); +} + +#endif +#endif + +#ifndef pv_display +#if defined(NEED_pv_display) +static char * DPPP_(my_pv_display)(pTHX_ SV * dsv, const char * pv, STRLEN cur, STRLEN len, STRLEN pvlim); +static +#else +extern char * DPPP_(my_pv_display)(pTHX_ SV * dsv, const char * pv, STRLEN cur, STRLEN len, STRLEN pvlim); +#endif + +#ifdef pv_display +# undef pv_display +#endif +#define pv_display(a,b,c,d,e) DPPP_(my_pv_display)(aTHX_ a,b,c,d,e) +#define Perl_pv_display DPPP_(my_pv_display) + +#if defined(NEED_pv_display) || defined(NEED_pv_display_GLOBAL) + +char * +DPPP_(my_pv_display)(pTHX_ SV *dsv, const char *pv, STRLEN cur, STRLEN len, STRLEN pvlim) +{ + pv_pretty(dsv, pv, cur, pvlim, NULL, NULL, PERL_PV_PRETTY_DUMP); + if (len > cur && pv[cur] == '\0') + sv_catpvs(dsv, "\\0"); + return SvPVX(dsv); +} + +#endif +#endif + +#endif /* _P_P_PORTABILITY_H_ */ + +/* End of File ppport.h */ diff --git a/perl/t/00_compile.t b/perl/t/00_compile.t new file mode 100644 index 0000000..66fe8f0 --- /dev/null +++ b/perl/t/00_compile.t @@ -0,0 +1,6 @@ +use strict; +use warnings; +use Test::More tests => 1; + +use_ok 'Data::MessagePack'; + diff --git a/perl/t/01_pack.t b/perl/t/01_pack.t new file mode 100644 index 0000000..0917f04 --- /dev/null +++ b/perl/t/01_pack.t @@ -0,0 +1,61 @@ +use t::Util; +use Test::More; +use Data::MessagePack; + +sub packit { + local $_ = unpack("H*", Data::MessagePack->pack($_[0])); + s/(..)/$1 /g; + s/ $//; + $_; +} + +sub pis ($$) { + is packit($_[0]), $_[1], 'dump ' . $_[1]; +} + +my @dat = ( + 0, '00', + 1, '01', + 127, '7f', + 128, 'cc 80', + 255, 'cc ff', + 256, 'cd 01 00', + 65535, 'cd ff ff', + 65536, 'ce 00 01 00 00', + -1, 'ff', + -32, 'e0', + -33, 'd0 df', + -128, 'd0 80', + -129, 'd1 ff 7f', + -32768, 'd1 80 00', + -32769, 'd2 ff ff 7f ff', + 1.0, 'cb 3f f0 00 00 00 00 00 00', + "", 'a0', + "a", 'a1 61', + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 'bf 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61', + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 'da 00 20 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61', + undef, 'c0', + Data::MessagePack::true(), 'c3', + Data::MessagePack::false(), 'c2', + [], '90', + [+[]], '91 90', + [[], undef], '92 90 c0', + {'a', 0}, '81 a1 61 00', + 8388608, 'ce 00 80 00 00', + + [undef, false, true], '93 c0 c2 c3', + ["", "a", "bc", "def"], '94 a0 a1 61 a2 62 63 a3 64 65 66', + [[], [[undef]]], '92 90 91 91 c0', + [undef, false, true], '93 c0 c2 c3', + [[0, 64, 127], [-32, -16, -1]], '92 93 00 40 7f 93 e0 f0 ff', + [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], '99 00 d0 80 ff 00 d1 80 00 ff 00 d2 80 00 00 00 ff', + 2147483648, 'ce 80 00 00 00', + -2147483648, 'd2 80 00 00 00', +); +@dat = (2147483648, 'ce 80 00 00 00'); +plan tests => 1*(scalar(@dat)/2); + +for (my $i=0; $iunpack($v); +} + +sub pis ($$) { + is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0]; +} + +my @dat = do 't/data.pl'; + +plan tests => 1*(scalar(@dat)/2); + +for (my $i=0; $inew; +sub unpackit { + my $v = $_[0]; + $v =~ s/ //g; + $v = pack 'H*', $v; + $up->reset; + my $ret = $up->execute($v, 0); + if ($ret != length($v)) { + fail "extra bytes"; + } + return $up->data; +} + +sub pis ($$) { + is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0]; +} + +my @dat = do 't/data.pl'; + +plan tests => 1*(scalar(@dat)/2) + 1; + +isa_ok $up, 'Data::MessagePack::Unpacker'; +for (my $i=0; $iunpack( + Data::MessagePack->pack($_[0]), + ); +} + +sub pis ($) { + is_deeply invert($_[0]), $_[0], 'dump ' . $_[0]; +} + +my @dat = do 't/data.pl'; + +plan tests => 1*(scalar(@dat)/2); + +for (my $i=0; $iimport; + warnings->import; + + no strict 'refs'; + *{"$pkg\::true"} = sub () { + Data::MessagePack::true() + }; + *{"$pkg\::false"} = sub () { + Data::MessagePack::false() + }; +} + +1; diff --git a/perl/t/data.pl b/perl/t/data.pl new file mode 100644 index 0000000..1a2b2b4 --- /dev/null +++ b/perl/t/data.pl @@ -0,0 +1,15 @@ +no warnings 'uninitialized'; # i need this. i need this. +( + '93 c0 c2 c3' => [undef, false, true], + '94 a0 a1 61 a2 62 63 a3 64 65 66', ["", "a", "bc", "def"], + '92 90 91 91 c0', [[], [[undef]]], + '93 c0 c2 c3', [undef, false, true], + 'ce 80 00 00 00', 2147483648, + '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff', [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + '92 93 00 40 7f 93 e0 f0 ff', [[0, 64, 127], [-32, -16, -1]], + '96 dc 00 00 dc 00 01 c0 dc 00 02 c2 c3 dd 00 00 00 00 dd 00 00 00 01 c0 dd 00 00 00 02 c2 c3', [[], [undef], [false, true], [], [undef], [false, true]], + '96 da 00 00 da 00 01 61 da 00 02 61 62 db 00 00 00 00 db 00 00 00 01 61 db 00 00 00 02 61 62', ["", "a", "ab", "", "a", "ab"], + '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff', [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], + '82 c2 81 c0 c0 c3 81 c0 80', {false,{undef,undef}, true,{undef,{}}}, + '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2', [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}], +) diff --git a/perl/unpack.c b/perl/unpack.c new file mode 100644 index 0000000..e5e069e --- /dev/null +++ b/perl/unpack.c @@ -0,0 +1,268 @@ +#ifdef __cplusplus +extern "C" { +#endif + +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "ppport.h" + +#ifdef __cplusplus +}; +#endif + +typedef struct { + int finished; + SV* source; +} unpack_user; + +#include "msgpack/unpack_define.h" + +#define msgpack_unpack_struct(name) \ + struct template ## name + +#define msgpack_unpack_func(ret, name) \ + ret template ## name + +#define msgpack_unpack_callback(name) \ + template_callback ## name + +#define msgpack_unpack_object SV* + +#define msgpack_unpack_user unpack_user + +struct template_context; +typedef struct template_context msgpack_unpack_t; + +static void template_init(msgpack_unpack_t* u); + +static SV* template_data(msgpack_unpack_t* u); + +static int template_execute(msgpack_unpack_t* u, + const char* data, size_t len, size_t* off); + +static inline SV* template_callback_root(unpack_user* u) +{ return &PL_sv_undef; } + +static inline int template_callback_uint8(unpack_user* u, uint8_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_uint16(unpack_user* u, uint16_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_uint32(unpack_user* u, uint32_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_int8(unpack_user* u, int8_t d, SV** o) +{ *o = newSViv((long)d); return 0; } + +static inline int template_callback_int16(unpack_user* u, int16_t d, SV** o) +{ *o = newSViv((long)d); return 0; } + +static inline int template_callback_int32(unpack_user* u, int32_t d, SV** o) +{ *o = newSViv((long)d); return 0; } + +static inline int template_callback_int64(unpack_user* u, int64_t d, SV** o) +{ *o = newSViv(d); return 0; } + +static inline int template_callback_float(unpack_user* u, float d, SV** o) +{ *o = newSVnv(d); return 0; } + +static inline int template_callback_double(unpack_user* u, double d, SV** o) +{ *o = newSVnv(d); return 0; } + +static inline int template_callback_nil(unpack_user* u, SV** o) +{ *o = &PL_sv_undef; return 0; } + +static inline int template_callback_true(unpack_user* u, SV** o) +{ *o = &PL_sv_yes; return 0; } + +static inline int template_callback_false(unpack_user* u, SV** o) +{ *o = &PL_sv_no; return 0;} + +static inline int template_callback_array(unpack_user* u, unsigned int n, SV** o) +{ AV* a = newAV(); *o = (SV*)newRV_noinc((SV*)a); av_extend(a, n); return 0; } + +static inline int template_callback_array_item(unpack_user* u, SV** c, SV* o) +{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] + +static inline int template_callback_map(unpack_user* u, unsigned int n, SV** o) +{ HV * h = newHV(); *o = newRV_noinc((SV*)h); return 0; } + +static inline int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v) +{ hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; } + +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o) +{ *o = (l == 0) ? newSVpv("", 0) : newSVpv(p, l); return 0; } + +#define UNPACKER(from, name) \ + msgpack_unpack_t *name; \ + name = INT2PTR(msgpack_unpack_t*, SvROK((from)) ? SvIV(SvRV((from))) : SvIV((from))); \ + if(name == NULL) { \ + Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be."); \ + } + +#include "msgpack/unpack_template.h" + +SV* _msgpack_unpack(SV* data, int limit) { + msgpack_unpack_t mp; + unpack_user u = {0, &PL_sv_undef}; + int ret; + size_t from = 0; + STRLEN dlen; + const char * dptr = SvPV_const(data, dlen); + + template_init(&mp); + mp.user = u; + + mp.user.source = data; + ret = template_execute(&mp, dptr, (size_t)dlen, &from); + mp.user.source = &PL_sv_undef; + + if(ret < 0) { + Perl_croak(aTHX_ "parse error."); + } else if(ret == 0) { + Perl_croak(aTHX_ "insufficient bytes."); + } else { + if(from < dlen) { + Perl_croak(aTHX_ "extra bytes."); + } + return template_data(&mp); + } +} + +XS(xs_unpack_limit) { + dXSARGS; + + if (items != 3) { + Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata', $limit)"); + } + + { + int limit = SvIV(ST(2)); + ST(0) = _msgpack_unpack(ST(1), limit); + } + XSRETURN(1); +} + + +XS(xs_unpack) { + dXSARGS; + msgpack_unpack_t mp; + + if (items != 2) { + Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata')"); + } + + { + ST(0) = _msgpack_unpack(ST(1), sv_len(ST(1))); + } + + XSRETURN(1); +} + +/* ------------------------------ stream -- */ + +static void _reset(SV* self) { + UNPACKER(self, mp); + template_init(mp); + unpack_user u = {0, &PL_sv_undef}; + mp->user = u; +} + +XS(xs_unpacker_new) { + dXSARGS; + SV* self = sv_newmortal(); + msgpack_unpack_t *mp; + + Newx(mp, 1, msgpack_unpack_t); + + sv_setref_pv(self, "Data::MessagePack::Unpacker", mp); + _reset(self); + + ST(0) = self; + XSRETURN(1); +} + +static SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) { + UNPACKER(self, mp); + + size_t from = off; + const char* dptr = SvPV_nolen_const(data); + long dlen = limit; + int ret; + + if(from >= dlen) { + Perl_croak(aTHX_ "offset is bigger than data buffer size."); + } + + mp->user.source = data; + ret = template_execute(mp, dptr, (size_t)dlen, &from); + mp->user.source = &PL_sv_undef; + + if(ret < 0) { + Perl_croak(aTHX_ "parse error."); + } else if(ret > 0) { + mp->user.finished = 1; + return newSVuv(from); + } else { + mp->user.finished = 0; + return newSVuv(from); + } +} + +XS(xs_unpacker_execute) { + dXSARGS; + SV* self = ST(0); + SV* data = ST(1); + IV off = SvIV(ST(2)); + + ST(0) = _execute_impl(self, data, off, sv_len(data)); + + XSRETURN(1); +} + +XS(xs_unpacker_execute_limit) { + dXSARGS; + SV* self = ST(0); + SV* data = ST(1); + IV off = SvIV(ST(2)); + IV limit = SvIV(ST(3)); + + ST(0) = _execute_impl(self, data, off, limit); + + XSRETURN(1); +} + +XS(xs_unpacker_is_finished) { + dXSARGS; + + UNPACKER(ST(0), mp); + ST(0) = (mp->user.finished) ? &PL_sv_yes : &PL_sv_no; + + XSRETURN(1); +} + +XS(xs_unpacker_data) { + dXSARGS; + + UNPACKER(ST(0), mp); + ST(0) = template_data(mp); + + XSRETURN(1); +} + +XS(xs_unpacker_reset) { + dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: $unpacker->reset()"); + } + + _reset(ST(0)); + + XSRETURN(0); +} + diff --git a/perl/xt/99_pod.t b/perl/xt/99_pod.t new file mode 100644 index 0000000..437887a --- /dev/null +++ b/perl/xt/99_pod.t @@ -0,0 +1,4 @@ +use Test::More; +eval "use Test::Pod 1.00"; +plan skip_all => "Test::Pod 1.00 required for testing POD" if $@; +all_pod_files_ok(); From 5710b87b0619b4754000a86da922708de40406af Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 13:09:05 +0900 Subject: [PATCH 0113/1172] perl: enhancement portability --- perl/MessagePack.c | 1 + perl/pack.c | 1 - perl/unpack.c | 4 +++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/perl/MessagePack.c b/perl/MessagePack.c index c40e46a..3b33fee 100644 --- a/perl/MessagePack.c +++ b/perl/MessagePack.c @@ -4,6 +4,7 @@ extern "C" { #include "EXTERN.h" #include "perl.h" #include "XSUB.h" +#define NEED_newCONSTSUB #include "ppport.h" #ifdef __cplusplus }; diff --git a/perl/pack.c b/perl/pack.c index 5bb667b..e9a5023 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -27,7 +27,6 @@ extern "C" { #define _PACK_WRAPPER(t) msgpack_pack_##t #define PACK_WRAPPER(t) _PACK_WRAPPER(t) -// move to pack.c static void _msgpack_pack_sv(SV* buf, SV* val) { if (val==NULL) { msgpack_pack_nil(buf); diff --git a/perl/unpack.c b/perl/unpack.c index e5e069e..fcbd2e2 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -5,6 +5,8 @@ extern "C" { #include "EXTERN.h" #include "perl.h" #include "XSUB.h" +#define NEED_newRV_noinc +#define NEED_sv_2pv_flags #include "ppport.h" #ifdef __cplusplus @@ -87,7 +89,7 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, SV** o { AV* a = newAV(); *o = (SV*)newRV_noinc((SV*)a); av_extend(a, n); return 0; } static inline int template_callback_array_item(unpack_user* u, SV** c, SV* o) -{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] +{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */ static inline int template_callback_map(unpack_user* u, unsigned int n, SV** o) { HV * h = newHV(); *o = newRV_noinc((SV*)h); return 0; } From c7b6bb730362f674ac573f4efae8169544a113a2 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 13:09:05 +0900 Subject: [PATCH 0114/1172] perl: enhancement portability --- perl/MessagePack.c | 1 + perl/pack.c | 1 - perl/unpack.c | 4 +++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/perl/MessagePack.c b/perl/MessagePack.c index c40e46a..3b33fee 100644 --- a/perl/MessagePack.c +++ b/perl/MessagePack.c @@ -4,6 +4,7 @@ extern "C" { #include "EXTERN.h" #include "perl.h" #include "XSUB.h" +#define NEED_newCONSTSUB #include "ppport.h" #ifdef __cplusplus }; diff --git a/perl/pack.c b/perl/pack.c index 5bb667b..e9a5023 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -27,7 +27,6 @@ extern "C" { #define _PACK_WRAPPER(t) msgpack_pack_##t #define PACK_WRAPPER(t) _PACK_WRAPPER(t) -// move to pack.c static void _msgpack_pack_sv(SV* buf, SV* val) { if (val==NULL) { msgpack_pack_nil(buf); diff --git a/perl/unpack.c b/perl/unpack.c index e5e069e..fcbd2e2 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -5,6 +5,8 @@ extern "C" { #include "EXTERN.h" #include "perl.h" #include "XSUB.h" +#define NEED_newRV_noinc +#define NEED_sv_2pv_flags #include "ppport.h" #ifdef __cplusplus @@ -87,7 +89,7 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, SV** o { AV* a = newAV(); *o = (SV*)newRV_noinc((SV*)a); av_extend(a, n); return 0; } static inline int template_callback_array_item(unpack_user* u, SV** c, SV* o) -{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] +{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */ static inline int template_callback_map(unpack_user* u, unsigned int n, SV** o) { HV * h = newHV(); *o = newRV_noinc((SV*)h); return 0; } From 28e113fd006a213c56a8b01057ebf71570bee2b1 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 22:43:16 +0900 Subject: [PATCH 0115/1172] oops. remove debugging code. --- perl/t/01_pack.t | 1 - 1 file changed, 1 deletion(-) diff --git a/perl/t/01_pack.t b/perl/t/01_pack.t index 0917f04..60c67f8 100644 --- a/perl/t/01_pack.t +++ b/perl/t/01_pack.t @@ -52,7 +52,6 @@ my @dat = ( 2147483648, 'ce 80 00 00 00', -2147483648, 'd2 80 00 00 00', ); -@dat = (2147483648, 'ce 80 00 00 00'); plan tests => 1*(scalar(@dat)/2); for (my $i=0; $i Date: Wed, 15 Apr 2009 22:43:16 +0900 Subject: [PATCH 0116/1172] oops. remove debugging code. --- perl/t/01_pack.t | 1 - 1 file changed, 1 deletion(-) diff --git a/perl/t/01_pack.t b/perl/t/01_pack.t index 0917f04..60c67f8 100644 --- a/perl/t/01_pack.t +++ b/perl/t/01_pack.t @@ -52,7 +52,6 @@ my @dat = ( 2147483648, 'ce 80 00 00 00', -2147483648, 'd2 80 00 00 00', ); -@dat = (2147483648, 'ce 80 00 00 00'); plan tests => 1*(scalar(@dat)/2); for (my $i=0; $i Date: Wed, 15 Apr 2009 22:43:59 +0900 Subject: [PATCH 0117/1172] perl: use more efficent strategy for memory allocation.this code taken from JSON::XS. thanks to mlehmann++ --- perl/pack.c | 85 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 60 insertions(+), 25 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index e9a5023..bd0c715 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -1,3 +1,7 @@ +/* + * code is written by tokuhirom. + * buffer alocation technique is taken from JSON::XS. thanks to mlehmann. + */ #ifdef __cplusplus extern "C" { #endif @@ -17,43 +21,64 @@ extern "C" { #define msgpack_pack_inline_func_cint(name) \ static inline void msgpack_pack ## name -#define msgpack_pack_user SV* +typedef struct { + char *cur; /* SvPVX (sv) + current output position */ + char *end; /* SvEND (sv) */ + SV *sv; /* result scalar */ +} enc_t; +void need(enc_t *enc, STRLEN len); -#define msgpack_pack_append_buffer(user, buf, len) \ - sv_catpvn(user, (const char*)(buf), len); +#define msgpack_pack_user enc_t* + +#define msgpack_pack_append_buffer(enc, buf, len) \ + need(enc, len); \ + memcpy(enc->cur, buf, len); \ + enc->cur += len; #include "msgpack/pack_template.h" #define _PACK_WRAPPER(t) msgpack_pack_##t #define PACK_WRAPPER(t) _PACK_WRAPPER(t) +#define INIT_SIZE 32 /* initial scalar size to be allocated */ -static void _msgpack_pack_sv(SV* buf, SV* val) { +void need(enc_t *enc, STRLEN len) +{ + if (enc->cur + len >= enc->end) { + STRLEN cur = enc->cur - (char *)SvPVX (enc->sv); + SvGROW (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1); + enc->cur = SvPVX (enc->sv) + cur; + enc->end = SvPVX (enc->sv) + SvLEN (enc->sv) - 1; + } +} + +static void _msgpack_pack_sv(enc_t *enc, SV* val) { if (val==NULL) { - msgpack_pack_nil(buf); + msgpack_pack_nil(enc); return; } switch (SvTYPE(val)) { case SVt_NULL: - msgpack_pack_nil(buf); + msgpack_pack_nil(enc); break; case SVt_IV: if (SvIOK_UV(val)) { - msgpack_pack_uint32(buf, SvUV(val)); + msgpack_pack_uint32(enc, SvUV(val)); } else { - PACK_WRAPPER(IVTYPE)(buf, SvIV(val)); + PACK_WRAPPER(IVTYPE)(enc, SvIV(val)); } break; case SVt_PVNV: { STRLEN len = 0; + need(enc, 1); char *pv = SvPV(val, len); if (len == 1 && *pv == '1') { - msgpack_pack_true(buf); + msgpack_pack_true(enc); } else if (len == 0 && *pv==0) { - msgpack_pack_false(buf); + msgpack_pack_false(enc); } else { - msgpack_pack_nil(buf); + msgpack_pack_nil(enc); } } break; @@ -61,25 +86,26 @@ static void _msgpack_pack_sv(SV* buf, SV* val) { { STRLEN len; char * cval = SvPV(val, len); - msgpack_pack_raw(buf, len); - msgpack_pack_raw_body(buf, cval, len); + msgpack_pack_raw(enc, len); + msgpack_pack_raw_body(enc, cval, len); } break; case SVt_NV: - PACK_WRAPPER(NVTYPE)(buf, SvNV(val)); + PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); break; case SVt_PVAV: { AV* ary = (AV*)val; int len = av_len(ary) + 1; int i; - msgpack_pack_array(buf, len); + need(enc, 1); + msgpack_pack_array(enc, len); for (i=0; i Date: Wed, 15 Apr 2009 22:43:59 +0900 Subject: [PATCH 0118/1172] perl: use more efficent strategy for memory allocation.this code taken from JSON::XS. thanks to mlehmann++ --- perl/pack.c | 85 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 60 insertions(+), 25 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index e9a5023..bd0c715 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -1,3 +1,7 @@ +/* + * code is written by tokuhirom. + * buffer alocation technique is taken from JSON::XS. thanks to mlehmann. + */ #ifdef __cplusplus extern "C" { #endif @@ -17,43 +21,64 @@ extern "C" { #define msgpack_pack_inline_func_cint(name) \ static inline void msgpack_pack ## name -#define msgpack_pack_user SV* +typedef struct { + char *cur; /* SvPVX (sv) + current output position */ + char *end; /* SvEND (sv) */ + SV *sv; /* result scalar */ +} enc_t; +void need(enc_t *enc, STRLEN len); -#define msgpack_pack_append_buffer(user, buf, len) \ - sv_catpvn(user, (const char*)(buf), len); +#define msgpack_pack_user enc_t* + +#define msgpack_pack_append_buffer(enc, buf, len) \ + need(enc, len); \ + memcpy(enc->cur, buf, len); \ + enc->cur += len; #include "msgpack/pack_template.h" #define _PACK_WRAPPER(t) msgpack_pack_##t #define PACK_WRAPPER(t) _PACK_WRAPPER(t) +#define INIT_SIZE 32 /* initial scalar size to be allocated */ -static void _msgpack_pack_sv(SV* buf, SV* val) { +void need(enc_t *enc, STRLEN len) +{ + if (enc->cur + len >= enc->end) { + STRLEN cur = enc->cur - (char *)SvPVX (enc->sv); + SvGROW (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1); + enc->cur = SvPVX (enc->sv) + cur; + enc->end = SvPVX (enc->sv) + SvLEN (enc->sv) - 1; + } +} + +static void _msgpack_pack_sv(enc_t *enc, SV* val) { if (val==NULL) { - msgpack_pack_nil(buf); + msgpack_pack_nil(enc); return; } switch (SvTYPE(val)) { case SVt_NULL: - msgpack_pack_nil(buf); + msgpack_pack_nil(enc); break; case SVt_IV: if (SvIOK_UV(val)) { - msgpack_pack_uint32(buf, SvUV(val)); + msgpack_pack_uint32(enc, SvUV(val)); } else { - PACK_WRAPPER(IVTYPE)(buf, SvIV(val)); + PACK_WRAPPER(IVTYPE)(enc, SvIV(val)); } break; case SVt_PVNV: { STRLEN len = 0; + need(enc, 1); char *pv = SvPV(val, len); if (len == 1 && *pv == '1') { - msgpack_pack_true(buf); + msgpack_pack_true(enc); } else if (len == 0 && *pv==0) { - msgpack_pack_false(buf); + msgpack_pack_false(enc); } else { - msgpack_pack_nil(buf); + msgpack_pack_nil(enc); } } break; @@ -61,25 +86,26 @@ static void _msgpack_pack_sv(SV* buf, SV* val) { { STRLEN len; char * cval = SvPV(val, len); - msgpack_pack_raw(buf, len); - msgpack_pack_raw_body(buf, cval, len); + msgpack_pack_raw(enc, len); + msgpack_pack_raw_body(enc, cval, len); } break; case SVt_NV: - PACK_WRAPPER(NVTYPE)(buf, SvNV(val)); + PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); break; case SVt_PVAV: { AV* ary = (AV*)val; int len = av_len(ary) + 1; int i; - msgpack_pack_array(buf, len); + need(enc, 1); + msgpack_pack_array(enc, len); for (i=0; i Date: Wed, 15 Apr 2009 23:02:27 +0900 Subject: [PATCH 0119/1172] perl: renamed benchmark script --- perl/benchmark/{p1.pl => serialize.pl} | 2 ++ 1 file changed, 2 insertions(+) rename perl/benchmark/{p1.pl => serialize.pl} (73%) diff --git a/perl/benchmark/p1.pl b/perl/benchmark/serialize.pl similarity index 73% rename from perl/benchmark/p1.pl rename to perl/benchmark/serialize.pl index 257932e..626ae03 100644 --- a/perl/benchmark/p1.pl +++ b/perl/benchmark/serialize.pl @@ -7,6 +7,8 @@ use Benchmark ':all'; my $a = [0..2**24]; print "-- serialize\n"; +print "JSON::XS: $JSON::XS::VERSION\n"; +print "Data::MessagePack: $Data::MessagePack::VERSION\n"; cmpthese( -1 => { json => sub { JSON::XS::encode_json($a) }, From 0b083030b106113c4cc92d7b1d0cae4db1b8c00a Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 23:02:27 +0900 Subject: [PATCH 0120/1172] perl: renamed benchmark script --- perl/benchmark/{p1.pl => serialize.pl} | 2 ++ 1 file changed, 2 insertions(+) rename perl/benchmark/{p1.pl => serialize.pl} (73%) diff --git a/perl/benchmark/p1.pl b/perl/benchmark/serialize.pl similarity index 73% rename from perl/benchmark/p1.pl rename to perl/benchmark/serialize.pl index 257932e..626ae03 100644 --- a/perl/benchmark/p1.pl +++ b/perl/benchmark/serialize.pl @@ -7,6 +7,8 @@ use Benchmark ':all'; my $a = [0..2**24]; print "-- serialize\n"; +print "JSON::XS: $JSON::XS::VERSION\n"; +print "Data::MessagePack: $Data::MessagePack::VERSION\n"; cmpthese( -1 => { json => sub { JSON::XS::encode_json($a) }, From e0bd2a291182bb6cfe1971de59bcf18f7bd52b04 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 23:06:47 +0900 Subject: [PATCH 0121/1172] perl: added benchmark script for deserialization --- perl/benchmark/deserialize.pl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 perl/benchmark/deserialize.pl diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl new file mode 100644 index 0000000..fd21f08 --- /dev/null +++ b/perl/benchmark/deserialize.pl @@ -0,0 +1,20 @@ +use strict; +use warnings; +use Data::MessagePack; +use JSON::XS; +use Benchmark ':all'; + +my $a = [0..2**24]; +my $j = JSON::XS::encode_json($a); +my $m = Data::MessagePack->pack($a); + +print "-- deserialize\n"; +print "JSON::XS: $JSON::XS::VERSION\n"; +print "Data::MessagePack: $Data::MessagePack::VERSION\n"; +cmpthese( + -1 => { + json => sub { JSON::XS::decode_json($j) }, + mp => sub { Data::MessagePack->unpack($m) }, + } +); + From 45321baa669435e048f396b22c46efbbaf753b85 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 23:06:47 +0900 Subject: [PATCH 0122/1172] perl: added benchmark script for deserialization --- perl/benchmark/deserialize.pl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 perl/benchmark/deserialize.pl diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl new file mode 100644 index 0000000..fd21f08 --- /dev/null +++ b/perl/benchmark/deserialize.pl @@ -0,0 +1,20 @@ +use strict; +use warnings; +use Data::MessagePack; +use JSON::XS; +use Benchmark ':all'; + +my $a = [0..2**24]; +my $j = JSON::XS::encode_json($a); +my $m = Data::MessagePack->pack($a); + +print "-- deserialize\n"; +print "JSON::XS: $JSON::XS::VERSION\n"; +print "Data::MessagePack: $Data::MessagePack::VERSION\n"; +cmpthese( + -1 => { + json => sub { JSON::XS::decode_json($j) }, + mp => sub { Data::MessagePack->unpack($m) }, + } +); + From b140b27b9af50fe3216a7c5571cb9c957a3259ee Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 23:11:26 +0900 Subject: [PATCH 0123/1172] perl: added argument check --- perl/pack.c | 4 +++- perl/unpack.c | 28 ++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index bd0c715..d30735e 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -136,7 +136,9 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { XS(xs_pack) { dXSARGS; - PERL_UNUSED_VAR(items); /* TODO: check argument count */ + if (items != 2) { + Perl_croak(aTHX_ "Usage: Data::MessagePack->pack($dat)"); + } SV* val = ST(1); diff --git a/perl/unpack.c b/perl/unpack.c index fcbd2e2..c2e32dd 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -177,6 +177,10 @@ static void _reset(SV* self) { XS(xs_unpacker_new) { dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: Data::MessagePack::Unpacker->new()"); + } + SV* self = sv_newmortal(); msgpack_unpack_t *mp; @@ -218,17 +222,27 @@ static SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) { XS(xs_unpacker_execute) { dXSARGS; - SV* self = ST(0); - SV* data = ST(1); - IV off = SvIV(ST(2)); + if (items != 3) { + Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off)"); + } - ST(0) = _execute_impl(self, data, off, sv_len(data)); + { + SV* self = ST(0); + SV* data = ST(1); + IV off = SvIV(ST(2)); + + ST(0) = _execute_impl(self, data, off, sv_len(data)); + } XSRETURN(1); } XS(xs_unpacker_execute_limit) { dXSARGS; + if (items != 4) { + Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off, limit)"); + } + SV* self = ST(0); SV* data = ST(1); IV off = SvIV(ST(2)); @@ -241,6 +255,9 @@ XS(xs_unpacker_execute_limit) { XS(xs_unpacker_is_finished) { dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: $unpacker->is_finished()"); + } UNPACKER(ST(0), mp); ST(0) = (mp->user.finished) ? &PL_sv_yes : &PL_sv_no; @@ -250,6 +267,9 @@ XS(xs_unpacker_is_finished) { XS(xs_unpacker_data) { dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: $unpacker->data()"); + } UNPACKER(ST(0), mp); ST(0) = template_data(mp); From 0b3db489768ed91e42eaaccce2657abacf8972ab Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 23:11:26 +0900 Subject: [PATCH 0124/1172] perl: added argument check --- perl/pack.c | 4 +++- perl/unpack.c | 28 ++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index bd0c715..d30735e 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -136,7 +136,9 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { XS(xs_pack) { dXSARGS; - PERL_UNUSED_VAR(items); /* TODO: check argument count */ + if (items != 2) { + Perl_croak(aTHX_ "Usage: Data::MessagePack->pack($dat)"); + } SV* val = ST(1); diff --git a/perl/unpack.c b/perl/unpack.c index fcbd2e2..c2e32dd 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -177,6 +177,10 @@ static void _reset(SV* self) { XS(xs_unpacker_new) { dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: Data::MessagePack::Unpacker->new()"); + } + SV* self = sv_newmortal(); msgpack_unpack_t *mp; @@ -218,17 +222,27 @@ static SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) { XS(xs_unpacker_execute) { dXSARGS; - SV* self = ST(0); - SV* data = ST(1); - IV off = SvIV(ST(2)); + if (items != 3) { + Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off)"); + } - ST(0) = _execute_impl(self, data, off, sv_len(data)); + { + SV* self = ST(0); + SV* data = ST(1); + IV off = SvIV(ST(2)); + + ST(0) = _execute_impl(self, data, off, sv_len(data)); + } XSRETURN(1); } XS(xs_unpacker_execute_limit) { dXSARGS; + if (items != 4) { + Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off, limit)"); + } + SV* self = ST(0); SV* data = ST(1); IV off = SvIV(ST(2)); @@ -241,6 +255,9 @@ XS(xs_unpacker_execute_limit) { XS(xs_unpacker_is_finished) { dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: $unpacker->is_finished()"); + } UNPACKER(ST(0), mp); ST(0) = (mp->user.finished) ? &PL_sv_yes : &PL_sv_no; @@ -250,6 +267,9 @@ XS(xs_unpacker_is_finished) { XS(xs_unpacker_data) { dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: $unpacker->data()"); + } UNPACKER(ST(0), mp); ST(0) = template_data(mp); From b0062a7f6f47b9a4340d14c6fb0a3ae0f60c8259 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 23:14:56 +0900 Subject: [PATCH 0125/1172] perl: oops. this doens't needed. --- perl/pack.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index d30735e..9c32dc1 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -71,7 +71,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { case SVt_PVNV: { STRLEN len = 0; - need(enc, 1); char *pv = SvPV(val, len); if (len == 1 && *pv == '1') { msgpack_pack_true(enc); @@ -98,7 +97,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { AV* ary = (AV*)val; int len = av_len(ary) + 1; int i; - need(enc, 1); msgpack_pack_array(enc, len); for (i=0; i Date: Wed, 15 Apr 2009 23:14:56 +0900 Subject: [PATCH 0126/1172] perl: oops. this doens't needed. --- perl/pack.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index d30735e..9c32dc1 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -71,7 +71,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { case SVt_PVNV: { STRLEN len = 0; - need(enc, 1); char *pv = SvPV(val, len); if (len == 1 && *pv == '1') { msgpack_pack_true(enc); @@ -98,7 +97,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { AV* ary = (AV*)val; int len = av_len(ary) + 1; int i; - need(enc, 1); msgpack_pack_array(enc, len); for (i=0; i Date: Wed, 13 May 2009 17:53:27 +0900 Subject: [PATCH 0127/1172] cpp: const --- cpp/object.hpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 09ddb89..07917c2 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -80,13 +80,13 @@ struct object { type::object_type type; union_type via; - bool is_nil() { return type == type::NIL; } + bool is_nil() const { return type == type::NIL; } template - T as(); + T as() const; template - void convert(T* v); + void convert(T* v) const; object(); object(msgpack_object obj); @@ -96,7 +96,7 @@ private: struct implicit_type; public: - implicit_type convert(); + implicit_type convert() const; }; struct object_kv { @@ -201,25 +201,25 @@ inline object::operator msgpack_object() } -inline object::implicit_type object::convert() +inline object::implicit_type object::convert() const { return implicit_type(*this); } template -inline T object::as() +inline void object::convert(T* v) const +{ + *this >> *v; +} + +template +inline T object::as() const { T v; convert(&v); return v; } -template -inline void object::convert(T* v) -{ - *this >> *v; -} - // obsolete template From 8903e2dea9f7f719897ca2420fae9376982c2c3b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 13 May 2009 17:53:27 +0900 Subject: [PATCH 0128/1172] cpp: const --- cpp/object.hpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 09ddb89..07917c2 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -80,13 +80,13 @@ struct object { type::object_type type; union_type via; - bool is_nil() { return type == type::NIL; } + bool is_nil() const { return type == type::NIL; } template - T as(); + T as() const; template - void convert(T* v); + void convert(T* v) const; object(); object(msgpack_object obj); @@ -96,7 +96,7 @@ private: struct implicit_type; public: - implicit_type convert(); + implicit_type convert() const; }; struct object_kv { @@ -201,25 +201,25 @@ inline object::operator msgpack_object() } -inline object::implicit_type object::convert() +inline object::implicit_type object::convert() const { return implicit_type(*this); } template -inline T object::as() +inline void object::convert(T* v) const +{ + *this >> *v; +} + +template +inline T object::as() const { T v; convert(&v); return v; } -template -inline void object::convert(T* v) -{ - *this >> *v; -} - // obsolete template From c8bb68d4334977ddd725804e23b2def0435a8d99 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Fri, 22 May 2009 14:31:20 +0900 Subject: [PATCH 0129/1172] add pyx --- msgpack.pyx | 162 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 msgpack.pyx diff --git a/msgpack.pyx b/msgpack.pyx new file mode 100644 index 0000000..6885317 --- /dev/null +++ b/msgpack.pyx @@ -0,0 +1,162 @@ +# coding: utf-8 + + +cdef extern from "Python.h": + ctypedef char* const_char_ptr "const char*" + cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + +cdef extern from "stdlib.h": + void* malloc(int) + void free(void*) +cdef extern from "string.h": + int memcpy(char*dst, char*src, unsigned int size) + +cdef extern from "msgpack/pack.h": + ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) + + struct msgpack_packer: + void *data + msgpack_packer_write callback + + void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) + void msgpack_pack_int(msgpack_packer* pk, int d) + void msgpack_pack_nil(msgpack_packer* pk) + void msgpack_pack_true(msgpack_packer* pk) + void msgpack_pack_false(msgpack_packer* pk) + void msgpack_pack_long_long(msgpack_packer* pk, long long d) + void msgpack_pack_double(msgpack_packer* pk, double d) + void msgpack_pack_array(msgpack_packer* pk, size_t l) + void msgpack_pack_map(msgpack_packer* pk, size_t l) + void msgpack_pack_raw(msgpack_packer* pk, size_t l) + void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) + +cdef extern from "msgpack/unpack.h": + ctypedef struct msgpack_unpacker + + +cdef int BUFF_SIZE=2*1024 + +cdef class Packer: + cdef char* buff + cdef unsigned int length + cdef unsigned int allocated + cdef msgpack_packer pk + cdef object strm + + def __init__(self, strm, int size=0): + """Make packer that pack data into strm. + + strm must have `write(bytes)` method. + size specifies local buffer size. + """ + if size <= 0: + size = BUFF_SIZE + + self.strm = strm + self.buff = malloc(size) + self.allocated = size + self.length = 0 + + msgpack_packer_init(&self.pk, self, _packer_write) + + + def flush(self): + """Flash local buffer and output stream if it has 'flush()' method.""" + if self.length > 0: + self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + self.length = 0 + if hasattr(self.strm, 'flush'): + self.strm.flush() + + def pack_list(self, len): + """Start packing sequential objects. + + Example: + + packer.pack_list(2) + packer.pack('foo') + packer.pack('bar') + + This code is same as below code: + + packer.pack(['foo', 'bar']) + """ + msgpack_pack_array(&self.pk, len) + + def pack_dict(self, len): + """Start packing key-value objects. + + Example: + + packer.pack_dict(1) + packer.pack('foo') + packer.pack('bar') + + This code is same as below code: + + packer.pack({'foo', 'bar'}) + """ + msgpack_pack_map(&self.pk, len) + + def __call__(self, object o): + cdef long long intval + cdef double fval + cdef char* rawval + + if o is None: + msgpack_pack_nil(&self.pk) + elif o is True: + msgpack_pack_true(&self.pk) + elif o is False: + msgpack_pack_false(&self.pk) + elif isinstance(o, int): + intval = o + msgpack_pack_long_long(&self.pk, intval) + elif isinstance(o, float): + fval = 9 + msgpack_pack_double(&self.pk, fval) + elif isinstance(o, str): + rawval = o + msgpack_pack_raw(&self.pk, len(o)) + msgpack_pack_raw_body(&self.pk, rawval, len(o)) + elif isinstance(o, unicode): + # todo + pass + elif isinstance(o, dict): + msgpack_pack_map(&self.pk, len(o)) + for k,v in o.iteritems(): + self(k) + self(v) + elif isinstance(o, tuple): + msgpack_pack_array(&self.pk, len(o)) + for v in o: + self(v) + elif isinstance(o, list): + msgpack_pack_array(&self.pk, len(o)) + for v in o: + self(v) + elif hasattr(o, "__msgpack__"): + o.__msgpack__(self) + else: + raise TypeError, "can't serialize %r" % (o,) + +cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + if packer.length + l > packer.allocated: + if packer.length > 0: + packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + if l > 64: + packer.strm.write(PyString_FromStringAndSize(b, l)) + packer.length = 0 + else: + memcpy(packer.buff, b, l) + packer.length = l + else: + memcpy(packer.buff + packer.length, b, l) + packer.length += l + return 0 + +cdef class Unpacker: + def __init__(self): + pass + def unpack(strm): + pass From 3628ea22d4079fa0a235a938eadf7b24943e326c Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Fri, 22 May 2009 14:31:20 +0900 Subject: [PATCH 0130/1172] add pyx --- python/msgpack.pyx | 162 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 python/msgpack.pyx diff --git a/python/msgpack.pyx b/python/msgpack.pyx new file mode 100644 index 0000000..6885317 --- /dev/null +++ b/python/msgpack.pyx @@ -0,0 +1,162 @@ +# coding: utf-8 + + +cdef extern from "Python.h": + ctypedef char* const_char_ptr "const char*" + cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + +cdef extern from "stdlib.h": + void* malloc(int) + void free(void*) +cdef extern from "string.h": + int memcpy(char*dst, char*src, unsigned int size) + +cdef extern from "msgpack/pack.h": + ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) + + struct msgpack_packer: + void *data + msgpack_packer_write callback + + void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) + void msgpack_pack_int(msgpack_packer* pk, int d) + void msgpack_pack_nil(msgpack_packer* pk) + void msgpack_pack_true(msgpack_packer* pk) + void msgpack_pack_false(msgpack_packer* pk) + void msgpack_pack_long_long(msgpack_packer* pk, long long d) + void msgpack_pack_double(msgpack_packer* pk, double d) + void msgpack_pack_array(msgpack_packer* pk, size_t l) + void msgpack_pack_map(msgpack_packer* pk, size_t l) + void msgpack_pack_raw(msgpack_packer* pk, size_t l) + void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) + +cdef extern from "msgpack/unpack.h": + ctypedef struct msgpack_unpacker + + +cdef int BUFF_SIZE=2*1024 + +cdef class Packer: + cdef char* buff + cdef unsigned int length + cdef unsigned int allocated + cdef msgpack_packer pk + cdef object strm + + def __init__(self, strm, int size=0): + """Make packer that pack data into strm. + + strm must have `write(bytes)` method. + size specifies local buffer size. + """ + if size <= 0: + size = BUFF_SIZE + + self.strm = strm + self.buff = malloc(size) + self.allocated = size + self.length = 0 + + msgpack_packer_init(&self.pk, self, _packer_write) + + + def flush(self): + """Flash local buffer and output stream if it has 'flush()' method.""" + if self.length > 0: + self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + self.length = 0 + if hasattr(self.strm, 'flush'): + self.strm.flush() + + def pack_list(self, len): + """Start packing sequential objects. + + Example: + + packer.pack_list(2) + packer.pack('foo') + packer.pack('bar') + + This code is same as below code: + + packer.pack(['foo', 'bar']) + """ + msgpack_pack_array(&self.pk, len) + + def pack_dict(self, len): + """Start packing key-value objects. + + Example: + + packer.pack_dict(1) + packer.pack('foo') + packer.pack('bar') + + This code is same as below code: + + packer.pack({'foo', 'bar'}) + """ + msgpack_pack_map(&self.pk, len) + + def __call__(self, object o): + cdef long long intval + cdef double fval + cdef char* rawval + + if o is None: + msgpack_pack_nil(&self.pk) + elif o is True: + msgpack_pack_true(&self.pk) + elif o is False: + msgpack_pack_false(&self.pk) + elif isinstance(o, int): + intval = o + msgpack_pack_long_long(&self.pk, intval) + elif isinstance(o, float): + fval = 9 + msgpack_pack_double(&self.pk, fval) + elif isinstance(o, str): + rawval = o + msgpack_pack_raw(&self.pk, len(o)) + msgpack_pack_raw_body(&self.pk, rawval, len(o)) + elif isinstance(o, unicode): + # todo + pass + elif isinstance(o, dict): + msgpack_pack_map(&self.pk, len(o)) + for k,v in o.iteritems(): + self(k) + self(v) + elif isinstance(o, tuple): + msgpack_pack_array(&self.pk, len(o)) + for v in o: + self(v) + elif isinstance(o, list): + msgpack_pack_array(&self.pk, len(o)) + for v in o: + self(v) + elif hasattr(o, "__msgpack__"): + o.__msgpack__(self) + else: + raise TypeError, "can't serialize %r" % (o,) + +cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + if packer.length + l > packer.allocated: + if packer.length > 0: + packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + if l > 64: + packer.strm.write(PyString_FromStringAndSize(b, l)) + packer.length = 0 + else: + memcpy(packer.buff, b, l) + packer.length = l + else: + memcpy(packer.buff + packer.length, b, l) + packer.length += l + return 0 + +cdef class Unpacker: + def __init__(self): + pass + def unpack(strm): + pass From 861721314965cf89beaa17f3dfcdce90ccd8d089 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 3 Jun 2009 22:01:27 +0900 Subject: [PATCH 0131/1172] cpp: fix map converter --- cpp/type/map.hpp | 10 +++++----- ruby/test_case.rb | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index c136d53..552de57 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -80,13 +80,13 @@ inline std::map operator>> (object o, std::map& v) for(; p != pend; ++p) { K key; p->key.convert(&key); - typename std::map::iterator it(v.find(key)); - if(it != v.end()) { + typename std::map::iterator it(v.lower_bound(key)); + if(it != v.end() && !(key < it->first)) { + p->val.convert(&it->second); + } else { V val; p->val.convert(&val); - it->insert( std::pair(key, val) ); - } else { - p->val.convert(&it->second); + v.insert(it, std::pair(key, val)); } } return v; diff --git a/ruby/test_case.rb b/ruby/test_case.rb index 2d897df..4fbcea3 100644 --- a/ruby/test_case.rb +++ b/ruby/test_case.rb @@ -219,6 +219,7 @@ class MessagePackTestFormat < Test::Unit::TestCase def match(obj, buf) assert_equal(obj.to_msgpack, buf) + assert_equal(MessagePack::unpack(buf), obj) end end From b3846a411fb148c30cf99c7297b70fe8ad3bfaa9 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 3 Jun 2009 22:01:27 +0900 Subject: [PATCH 0132/1172] cpp: fix map converter --- cpp/type/map.hpp | 10 +++++----- ruby/test_case.rb | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index c136d53..552de57 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -80,13 +80,13 @@ inline std::map operator>> (object o, std::map& v) for(; p != pend; ++p) { K key; p->key.convert(&key); - typename std::map::iterator it(v.find(key)); - if(it != v.end()) { + typename std::map::iterator it(v.lower_bound(key)); + if(it != v.end() && !(key < it->first)) { + p->val.convert(&it->second); + } else { V val; p->val.convert(&val); - it->insert( std::pair(key, val) ); - } else { - p->val.convert(&it->second); + v.insert(it, std::pair(key, val)); } } return v; diff --git a/ruby/test_case.rb b/ruby/test_case.rb index 2d897df..4fbcea3 100644 --- a/ruby/test_case.rb +++ b/ruby/test_case.rb @@ -219,6 +219,7 @@ class MessagePackTestFormat < Test::Unit::TestCase def match(obj, buf) assert_equal(obj.to_msgpack, buf) + assert_equal(MessagePack::unpack(buf), obj) end end From 7cd359c1fdab21adb0fd5760741a6b1b0e4a806a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 7 Jun 2009 14:59:50 +0900 Subject: [PATCH 0133/1172] add msgpack_vrefbuffer, msgpack::vrefbuffer --- README | 2 +- c/Makefile.am | 2 + c/vrefbuffer.c | 135 +++++++++++++++++++++++++++++++++++++++++++++ c/vrefbuffer.h | 97 ++++++++++++++++++++++++++++++++ c/zone.c | 2 +- c/zone.h | 11 +++- configure.in | 2 +- cpp/Makefile.am | 1 + cpp/vrefbuffer.hpp | 85 ++++++++++++++++++++++++++++ cpp/zone.hpp.erb | 10 ++++ 10 files changed, 341 insertions(+), 6 deletions(-) create mode 100644 c/vrefbuffer.c create mode 100644 c/vrefbuffer.h create mode 100644 cpp/vrefbuffer.hpp diff --git a/README b/README index 0c1d440..2a1a3e0 100644 --- a/README +++ b/README @@ -8,7 +8,7 @@ Binary-based efficient data interchange format. MessagePack is only tested on Linux and Mac OS X, but it may run on other UNIX-like platforms. - Following programs is required to build: + Following programs are required to build: - gcc >= 4.1 with C++ support - ruby >= 1.8 (ruby is used as a preprocessor) diff --git a/c/Makefile.am b/c/Makefile.am index e7cdc10..daa8a76 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -3,11 +3,13 @@ lib_LTLIBRARIES = libmsgpackc.la libmsgpackc_la_SOURCES = \ unpack.c \ object.c \ + vrefbuffer.c \ zone.c nobase_include_HEADERS = \ msgpack.h \ msgpack/sbuffer.h \ + msgpack/vrefbuffer.h \ msgpack/pack.h \ msgpack/unpack.h \ msgpack/object.h \ diff --git a/c/vrefbuffer.c b/c/vrefbuffer.c new file mode 100644 index 0000000..bbaf61d --- /dev/null +++ b/c/vrefbuffer.c @@ -0,0 +1,135 @@ +/* + * MessagePack for C zero-copy buffer implementation + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/vrefbuffer.h" +#include +#include + +bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, + size_t ref_size, size_t chunk_size) +{ + if(chunk_size < sizeof(msgpack_vrefbuffer_chunk)+72) { + chunk_size = 72; + } else { + chunk_size -= sizeof(msgpack_vrefbuffer_chunk); + } + + vbuf->chunk_size = chunk_size; + vbuf->ref_size = ref_size; + + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + size_t nfirst = (sizeof(struct iovec) < 72/2) ? + 72 / sizeof(struct iovec) : 8; + + struct iovec* array = (struct iovec*)malloc( + sizeof(struct iovec) * nfirst); + if(array == NULL) { + return false; + } + + vbuf->tail = array; + vbuf->end = array + nfirst; + vbuf->array = array; + + vbuf->chunk = (msgpack_vrefbuffer_chunk*)malloc( + chunk_size + sizeof(msgpack_vrefbuffer_chunk)); + if(vbuf->chunk == NULL) { + free(array); + return false; + } + + vbuf->chunk->next = NULL; + vbuf->chunk->free = chunk_size; + + return true; +} + +void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf) +{ + msgpack_vrefbuffer_chunk* c = vbuf->chunk; + while(true) { + msgpack_vrefbuffer_chunk* n = c->next; + free(c); + if(n) { + c = n; + } else { + break; + } + } + free(vbuf->array); +} + +int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len) +{ + if(vbuf->tail == vbuf->end) { + const size_t nused = vbuf->end - vbuf->array; + const size_t nnext = nused * 2; + + struct iovec* nvec = (struct iovec*)realloc( + vbuf->array, sizeof(struct iovec)*nnext); + if(nvec == NULL) { + return -1; + } + + vbuf->array = nvec; + vbuf->end = nvec + nnext; + vbuf->tail = nvec + nused; + } + + vbuf->tail->iov_base = (char*)buf; + vbuf->tail->iov_len = len; + ++vbuf->tail; + + return 0; +} + +int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len) +{ + msgpack_vrefbuffer_chunk* chunk = vbuf->chunk; + size_t cur_size = vbuf->chunk_size; + + if(chunk->free < len) { + cur_size = (cur_size > len) ? cur_size : len; + + chunk = (msgpack_vrefbuffer_chunk*)malloc( + cur_size + sizeof(msgpack_vrefbuffer_chunk)); + if(chunk == NULL) { + return -1; + } + + chunk->free = cur_size; + chunk->next = vbuf->chunk; + vbuf->chunk = chunk; + } + + char* m = ((char*)chunk) + sizeof(msgpack_vrefbuffer_chunk) + + (cur_size - chunk->free); + + memcpy(m, buf, len); + chunk->free -= len; + + if(vbuf->tail != vbuf->array && m == + (const char*)((vbuf->tail-1)->iov_base) + (vbuf->tail-1)->iov_len) { + (vbuf->tail-1)->iov_len += len; + return 0; + } else { + return msgpack_vrefbuffer_append_ref(vbuf, m, len); + } +} + diff --git a/c/vrefbuffer.h b/c/vrefbuffer.h new file mode 100644 index 0000000..baa7c03 --- /dev/null +++ b/c/vrefbuffer.h @@ -0,0 +1,97 @@ +/* + * MessagePack for C zero-copy buffer implementation + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_VREFBUFFER_H__ +#define MSGPACK_VREFBUFFER_H__ + +#include "msgpack/zone.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifndef MSGPACK_VREFBUFFER_REF_SIZE +#define MSGPACK_VREFBUFFER_REF_SIZE 32 +#endif + +#ifndef MSGPACK_VREFBUFFER_CHUNK_SIZE +#define MSGPACK_VREFBUFFER_CHUNK_SIZE 2048 +#endif + +typedef struct msgpack_vrefbuffer_chunk { + size_t free; + struct msgpack_vrefbuffer_chunk* next; + /* data ... */ +} msgpack_vrefbuffer_chunk; + +typedef struct msgpack_vrefbuffer { + size_t chunk_size; + size_t ref_size; + + struct iovec* tail; + struct iovec* end; + struct iovec* array; + + msgpack_vrefbuffer_chunk* chunk; +} msgpack_vrefbuffer; + + +bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, + size_t ref_size, size_t chunk_size); +void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf); + +static inline int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len); + +static inline const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref); +static inline size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref); + +int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len); + +int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len); + + +int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len) +{ + msgpack_vrefbuffer* vbuf = (msgpack_vrefbuffer*)data; + + if(len < vbuf->ref_size) { + return msgpack_vrefbuffer_append_copy(vbuf, buf, len); + } else { + return msgpack_vrefbuffer_append_ref(vbuf, buf, len); + } +} + +const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref) +{ + return vref->array; +} + +size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref) +{ + return vref->tail - vref->array; +} + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/vrefbuffer.h */ + diff --git a/c/zone.c b/c/zone.c index 1aaad9f..79e90dc 100644 --- a/c/zone.c +++ b/c/zone.c @@ -27,7 +27,7 @@ static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_s msgpack_zone_chunk* array = (msgpack_zone_chunk*)malloc( sizeof(msgpack_zone_chunk) * nfirst); - if(!array) { + if(array == NULL) { return false; } diff --git a/c/zone.h b/c/zone.h index a3dfe41..79b51f8 100644 --- a/c/zone.h +++ b/c/zone.h @@ -66,6 +66,7 @@ msgpack_zone* msgpack_zone_new(size_t chunk_size); void msgpack_zone_free(msgpack_zone* zone); static inline void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); +static inline void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size); static inline bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data); @@ -82,10 +83,8 @@ void msgpack_zone_clear(msgpack_zone* zone); void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size); -void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size) { - size = ((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1); - msgpack_zone_chunk* chunk = zone->chunk_array.tail; if(chunk->free < size) { @@ -99,6 +98,12 @@ void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) return ptr; } +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +{ + return msgpack_zone_malloc_no_align(zone, + ((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1)); +} + bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, void (*func)(void* data), void* data); diff --git a/configure.in b/configure.in index 3569244..47b28cf 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.1) +AM_INIT_AUTOMAKE(msgpack, 0.3.2) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 76770f4..b9126f8 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -6,6 +6,7 @@ libmsgpack_la_SOURCES = \ nobase_include_HEADERS = \ msgpack.hpp \ msgpack/sbuffer.hpp \ + msgpack/vrefbuffer.hpp \ msgpack/pack.hpp \ msgpack/unpack.hpp \ msgpack/object.hpp \ diff --git a/cpp/vrefbuffer.hpp b/cpp/vrefbuffer.hpp new file mode 100644 index 0000000..549d77f --- /dev/null +++ b/cpp/vrefbuffer.hpp @@ -0,0 +1,85 @@ +// +// MessagePack for C++ zero-copy buffer implementation +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_VREFBUFFER_HPP__ +#define MSGPACK_VREFBUFFER_HPP__ + +#include "msgpack/vrefbuffer.h" +#include + +namespace msgpack { + + +class vrefbuffer : public msgpack_vrefbuffer { +public: + vrefbuffer(size_t ref_size = MSGPACK_VREFBUFFER_REF_SIZE, + size_t chunk_size = MSGPACK_VREFBUFFER_CHUNK_SIZE) + { + msgpack_vrefbuffer_init(this, ref_size, chunk_size); + } + + ~vrefbuffer() + { + msgpack_vrefbuffer_destroy(this); + } + +public: + void write(const char* buf, unsigned int len) + { + if(len < base::ref_size) { + append_copy(buf, len); + } else { + append_ref(buf, len); + } + } + + void append_ref(const char* buf, size_t len) + { + if(msgpack_vrefbuffer_append_ref(this, buf, len) < 0) { + throw std::bad_alloc(); + } + } + + void append_copy(const char* buf, size_t len) + { + if(msgpack_vrefbuffer_append_copy(this, buf, len) < 0) { + throw std::bad_alloc(); + } + } + + const struct iovec* vector() const + { + return msgpack_vrefbuffer_vec(this); + } + + size_t vector_size() const + { + return msgpack_vrefbuffer_veclen(this); + } + +private: + typedef msgpack_vrefbuffer base; + +private: + vrefbuffer(const vrefbuffer&); +}; + + +} // namespace msgpack + +#endif /* msgpack/vrefbuffer.hpp */ + diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 9e85080..8fd14a6 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -34,6 +34,7 @@ public: public: void* malloc(size_t size); + void* malloc_no_align(size_t size); void push_finalizer(void (*func)(void*), void* data); @@ -77,6 +78,15 @@ inline void* zone::malloc(size_t size) return ptr; } +inline void* zone::malloc_no_align(size_t size) +{ + void* ptr = msgpack_zone_malloc_no_align(this, size); + if(!ptr) { + throw std::bad_alloc(); + } + return ptr; +} + inline void zone::push_finalizer(void (*func)(void*), void* data) { if(!msgpack_zone_push_finalizer(this, func, data)) { From 8ed1d61529cb7d7e9577e648773fd8c91d87b9cf Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 7 Jun 2009 14:59:50 +0900 Subject: [PATCH 0134/1172] add msgpack_vrefbuffer, msgpack::vrefbuffer --- README | 2 +- c/Makefile.am | 2 + c/vrefbuffer.c | 135 +++++++++++++++++++++++++++++++++++++++++++++ c/vrefbuffer.h | 97 ++++++++++++++++++++++++++++++++ c/zone.c | 2 +- c/zone.h | 11 +++- configure.in | 2 +- cpp/Makefile.am | 1 + cpp/vrefbuffer.hpp | 85 ++++++++++++++++++++++++++++ cpp/zone.hpp.erb | 10 ++++ 10 files changed, 341 insertions(+), 6 deletions(-) create mode 100644 c/vrefbuffer.c create mode 100644 c/vrefbuffer.h create mode 100644 cpp/vrefbuffer.hpp diff --git a/README b/README index 0c1d440..2a1a3e0 100644 --- a/README +++ b/README @@ -8,7 +8,7 @@ Binary-based efficient data interchange format. MessagePack is only tested on Linux and Mac OS X, but it may run on other UNIX-like platforms. - Following programs is required to build: + Following programs are required to build: - gcc >= 4.1 with C++ support - ruby >= 1.8 (ruby is used as a preprocessor) diff --git a/c/Makefile.am b/c/Makefile.am index e7cdc10..daa8a76 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -3,11 +3,13 @@ lib_LTLIBRARIES = libmsgpackc.la libmsgpackc_la_SOURCES = \ unpack.c \ object.c \ + vrefbuffer.c \ zone.c nobase_include_HEADERS = \ msgpack.h \ msgpack/sbuffer.h \ + msgpack/vrefbuffer.h \ msgpack/pack.h \ msgpack/unpack.h \ msgpack/object.h \ diff --git a/c/vrefbuffer.c b/c/vrefbuffer.c new file mode 100644 index 0000000..bbaf61d --- /dev/null +++ b/c/vrefbuffer.c @@ -0,0 +1,135 @@ +/* + * MessagePack for C zero-copy buffer implementation + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/vrefbuffer.h" +#include +#include + +bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, + size_t ref_size, size_t chunk_size) +{ + if(chunk_size < sizeof(msgpack_vrefbuffer_chunk)+72) { + chunk_size = 72; + } else { + chunk_size -= sizeof(msgpack_vrefbuffer_chunk); + } + + vbuf->chunk_size = chunk_size; + vbuf->ref_size = ref_size; + + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + size_t nfirst = (sizeof(struct iovec) < 72/2) ? + 72 / sizeof(struct iovec) : 8; + + struct iovec* array = (struct iovec*)malloc( + sizeof(struct iovec) * nfirst); + if(array == NULL) { + return false; + } + + vbuf->tail = array; + vbuf->end = array + nfirst; + vbuf->array = array; + + vbuf->chunk = (msgpack_vrefbuffer_chunk*)malloc( + chunk_size + sizeof(msgpack_vrefbuffer_chunk)); + if(vbuf->chunk == NULL) { + free(array); + return false; + } + + vbuf->chunk->next = NULL; + vbuf->chunk->free = chunk_size; + + return true; +} + +void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf) +{ + msgpack_vrefbuffer_chunk* c = vbuf->chunk; + while(true) { + msgpack_vrefbuffer_chunk* n = c->next; + free(c); + if(n) { + c = n; + } else { + break; + } + } + free(vbuf->array); +} + +int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len) +{ + if(vbuf->tail == vbuf->end) { + const size_t nused = vbuf->end - vbuf->array; + const size_t nnext = nused * 2; + + struct iovec* nvec = (struct iovec*)realloc( + vbuf->array, sizeof(struct iovec)*nnext); + if(nvec == NULL) { + return -1; + } + + vbuf->array = nvec; + vbuf->end = nvec + nnext; + vbuf->tail = nvec + nused; + } + + vbuf->tail->iov_base = (char*)buf; + vbuf->tail->iov_len = len; + ++vbuf->tail; + + return 0; +} + +int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len) +{ + msgpack_vrefbuffer_chunk* chunk = vbuf->chunk; + size_t cur_size = vbuf->chunk_size; + + if(chunk->free < len) { + cur_size = (cur_size > len) ? cur_size : len; + + chunk = (msgpack_vrefbuffer_chunk*)malloc( + cur_size + sizeof(msgpack_vrefbuffer_chunk)); + if(chunk == NULL) { + return -1; + } + + chunk->free = cur_size; + chunk->next = vbuf->chunk; + vbuf->chunk = chunk; + } + + char* m = ((char*)chunk) + sizeof(msgpack_vrefbuffer_chunk) + + (cur_size - chunk->free); + + memcpy(m, buf, len); + chunk->free -= len; + + if(vbuf->tail != vbuf->array && m == + (const char*)((vbuf->tail-1)->iov_base) + (vbuf->tail-1)->iov_len) { + (vbuf->tail-1)->iov_len += len; + return 0; + } else { + return msgpack_vrefbuffer_append_ref(vbuf, m, len); + } +} + diff --git a/c/vrefbuffer.h b/c/vrefbuffer.h new file mode 100644 index 0000000..baa7c03 --- /dev/null +++ b/c/vrefbuffer.h @@ -0,0 +1,97 @@ +/* + * MessagePack for C zero-copy buffer implementation + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_VREFBUFFER_H__ +#define MSGPACK_VREFBUFFER_H__ + +#include "msgpack/zone.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifndef MSGPACK_VREFBUFFER_REF_SIZE +#define MSGPACK_VREFBUFFER_REF_SIZE 32 +#endif + +#ifndef MSGPACK_VREFBUFFER_CHUNK_SIZE +#define MSGPACK_VREFBUFFER_CHUNK_SIZE 2048 +#endif + +typedef struct msgpack_vrefbuffer_chunk { + size_t free; + struct msgpack_vrefbuffer_chunk* next; + /* data ... */ +} msgpack_vrefbuffer_chunk; + +typedef struct msgpack_vrefbuffer { + size_t chunk_size; + size_t ref_size; + + struct iovec* tail; + struct iovec* end; + struct iovec* array; + + msgpack_vrefbuffer_chunk* chunk; +} msgpack_vrefbuffer; + + +bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, + size_t ref_size, size_t chunk_size); +void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf); + +static inline int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len); + +static inline const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref); +static inline size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref); + +int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len); + +int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len); + + +int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len) +{ + msgpack_vrefbuffer* vbuf = (msgpack_vrefbuffer*)data; + + if(len < vbuf->ref_size) { + return msgpack_vrefbuffer_append_copy(vbuf, buf, len); + } else { + return msgpack_vrefbuffer_append_ref(vbuf, buf, len); + } +} + +const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref) +{ + return vref->array; +} + +size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref) +{ + return vref->tail - vref->array; +} + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/vrefbuffer.h */ + diff --git a/c/zone.c b/c/zone.c index 1aaad9f..79e90dc 100644 --- a/c/zone.c +++ b/c/zone.c @@ -27,7 +27,7 @@ static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_s msgpack_zone_chunk* array = (msgpack_zone_chunk*)malloc( sizeof(msgpack_zone_chunk) * nfirst); - if(!array) { + if(array == NULL) { return false; } diff --git a/c/zone.h b/c/zone.h index a3dfe41..79b51f8 100644 --- a/c/zone.h +++ b/c/zone.h @@ -66,6 +66,7 @@ msgpack_zone* msgpack_zone_new(size_t chunk_size); void msgpack_zone_free(msgpack_zone* zone); static inline void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); +static inline void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size); static inline bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data); @@ -82,10 +83,8 @@ void msgpack_zone_clear(msgpack_zone* zone); void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size); -void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size) { - size = ((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1); - msgpack_zone_chunk* chunk = zone->chunk_array.tail; if(chunk->free < size) { @@ -99,6 +98,12 @@ void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) return ptr; } +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +{ + return msgpack_zone_malloc_no_align(zone, + ((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1)); +} + bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, void (*func)(void* data), void* data); diff --git a/configure.in b/configure.in index 3569244..47b28cf 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.1) +AM_INIT_AUTOMAKE(msgpack, 0.3.2) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 76770f4..b9126f8 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -6,6 +6,7 @@ libmsgpack_la_SOURCES = \ nobase_include_HEADERS = \ msgpack.hpp \ msgpack/sbuffer.hpp \ + msgpack/vrefbuffer.hpp \ msgpack/pack.hpp \ msgpack/unpack.hpp \ msgpack/object.hpp \ diff --git a/cpp/vrefbuffer.hpp b/cpp/vrefbuffer.hpp new file mode 100644 index 0000000..549d77f --- /dev/null +++ b/cpp/vrefbuffer.hpp @@ -0,0 +1,85 @@ +// +// MessagePack for C++ zero-copy buffer implementation +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_VREFBUFFER_HPP__ +#define MSGPACK_VREFBUFFER_HPP__ + +#include "msgpack/vrefbuffer.h" +#include + +namespace msgpack { + + +class vrefbuffer : public msgpack_vrefbuffer { +public: + vrefbuffer(size_t ref_size = MSGPACK_VREFBUFFER_REF_SIZE, + size_t chunk_size = MSGPACK_VREFBUFFER_CHUNK_SIZE) + { + msgpack_vrefbuffer_init(this, ref_size, chunk_size); + } + + ~vrefbuffer() + { + msgpack_vrefbuffer_destroy(this); + } + +public: + void write(const char* buf, unsigned int len) + { + if(len < base::ref_size) { + append_copy(buf, len); + } else { + append_ref(buf, len); + } + } + + void append_ref(const char* buf, size_t len) + { + if(msgpack_vrefbuffer_append_ref(this, buf, len) < 0) { + throw std::bad_alloc(); + } + } + + void append_copy(const char* buf, size_t len) + { + if(msgpack_vrefbuffer_append_copy(this, buf, len) < 0) { + throw std::bad_alloc(); + } + } + + const struct iovec* vector() const + { + return msgpack_vrefbuffer_vec(this); + } + + size_t vector_size() const + { + return msgpack_vrefbuffer_veclen(this); + } + +private: + typedef msgpack_vrefbuffer base; + +private: + vrefbuffer(const vrefbuffer&); +}; + + +} // namespace msgpack + +#endif /* msgpack/vrefbuffer.hpp */ + diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 9e85080..8fd14a6 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -34,6 +34,7 @@ public: public: void* malloc(size_t size); + void* malloc_no_align(size_t size); void push_finalizer(void (*func)(void*), void* data); @@ -77,6 +78,15 @@ inline void* zone::malloc(size_t size) return ptr; } +inline void* zone::malloc_no_align(size_t size) +{ + void* ptr = msgpack_zone_malloc_no_align(this, size); + if(!ptr) { + throw std::bad_alloc(); + } + return ptr; +} + inline void zone::push_finalizer(void (*func)(void*), void* data) { if(!msgpack_zone_push_finalizer(this, func, data)) { From 711e4817a5e35a32c4b577664d8096de00fa802b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 00:23:38 +0900 Subject: [PATCH 0135/1172] support packing long and tuple. add missing files. --- python/msgpack.pyx | 53 ++++-- python/pack.h | 130 +++++++++++++++ python/unpack.h | 397 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 563 insertions(+), 17 deletions(-) create mode 100644 python/pack.h create mode 100644 python/unpack.h diff --git a/python/msgpack.pyx b/python/msgpack.pyx index 6885317..ceae9b6 100644 --- a/python/msgpack.pyx +++ b/python/msgpack.pyx @@ -3,15 +3,17 @@ cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" + ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) cdef extern from "stdlib.h": void* malloc(int) void free(void*) + cdef extern from "string.h": int memcpy(char*dst, char*src, unsigned int size) -cdef extern from "msgpack/pack.h": +cdef extern from "pack.h": ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) struct msgpack_packer: @@ -30,7 +32,7 @@ cdef extern from "msgpack/pack.h": void msgpack_pack_raw(msgpack_packer* pk, size_t l) void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef extern from "msgpack/unpack.h": +cdef extern from "unpack.h": ctypedef struct msgpack_unpacker @@ -98,7 +100,7 @@ cdef class Packer: """ msgpack_pack_map(&self.pk, len) - def __call__(self, object o): + def pack(self, object o): cdef long long intval cdef double fval cdef char* rawval @@ -109,6 +111,9 @@ cdef class Packer: msgpack_pack_true(&self.pk) elif o is False: msgpack_pack_false(&self.pk) + elif isinstance(o, long): + intval = o + msgpack_pack_long_long(&self.pk, intval) elif isinstance(o, int): intval = o msgpack_pack_long_long(&self.pk, intval) @@ -120,24 +125,21 @@ cdef class Packer: msgpack_pack_raw(&self.pk, len(o)) msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif isinstance(o, unicode): - # todo - pass + o = o.encode('utf-8') + rawval = o + msgpack_pack_raw(&self.pk, len(o)) + msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif isinstance(o, dict): msgpack_pack_map(&self.pk, len(o)) for k,v in o.iteritems(): - self(k) - self(v) - elif isinstance(o, tuple): + self.pack(k) + self.pack(v) + elif isinstance(o, tuple) or isinstance(o, list): msgpack_pack_array(&self.pk, len(o)) for v in o: - self(v) - elif isinstance(o, list): - msgpack_pack_array(&self.pk, len(o)) - for v in o: - self(v) - elif hasattr(o, "__msgpack__"): - o.__msgpack__(self) + self.pack(v) else: + # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): @@ -155,8 +157,25 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): packer.length += l return 0 +cdef extern from "msgpack/zone.h": + ctypedef struct msgpack_zone + +cdef extern from "unpack.c": + ctypedef struct template_context: + pass + int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) + void template_init(template_context* ctx) + PyObject* template_data(template_context* ctx) + + cdef class Unpacker: def __init__(self): pass - def unpack(strm): - pass + + def unpack(self, bytes_): + cdef const_char_ptr p = bytes_ + cdef template_context ctx + cdef size_t off = 0 + template_init(&ctx) + template_execute(&ctx, p, len(bytes_), &off) + return template_data(&ctx) diff --git a/python/pack.h b/python/pack.h new file mode 100644 index 0000000..4a336d5 --- /dev/null +++ b/python/pack.h @@ -0,0 +1,130 @@ +/* + * MessagePack for Python packing routine + * + * Copyright (C) 2009 Naoki INADA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_H__ +#define MSGPACK_PACK_H__ + +#if _MSC_VER +typedef signed char uint8_t; +typedef unsigned char uint8_t; +typedef short int16_t; +typedef unsigned short uint16_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef long long int64_t; +typedef unsigned long long uint64_t; +#elif +#include +#endif + +#include +#include +#include "msgpack/pack_define.h" +#include "msgpack/object.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len); + +typedef struct msgpack_packer { + void* data; + msgpack_packer_write callback; +} msgpack_packer; + +static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); + +static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); +static void msgpack_packer_free(msgpack_packer* pk); + +static int msgpack_pack_short(msgpack_packer* pk, short d); +static int msgpack_pack_int(msgpack_packer* pk, int d); +static int msgpack_pack_long(msgpack_packer* pk, long d); +static int msgpack_pack_long_long(msgpack_packer* pk, long long d); +static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); +static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); +static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); +static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); + +static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); +static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); +static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); +static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); +static int msgpack_pack_int8(msgpack_packer* pk, int8_t d); +static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); +static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); +static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); + +static int msgpack_pack_float(msgpack_packer* pk, float d); +static int msgpack_pack_double(msgpack_packer* pk, double d); + +static int msgpack_pack_nil(msgpack_packer* pk); +static int msgpack_pack_true(msgpack_packer* pk); +static int msgpack_pack_false(msgpack_packer* pk); + +static int msgpack_pack_array(msgpack_packer* pk, unsigned int n); + +static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); + +static int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); + +int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); + + + +#define msgpack_pack_inline_func(name) \ + static inline int msgpack_pack ## name + +#define msgpack_pack_inline_func_cint(name) \ + static inline int msgpack_pack ## name + +#define msgpack_pack_user msgpack_packer* + +#define msgpack_pack_append_buffer(user, buf, len) \ + return (*(user)->callback)((user)->data, (const char*)buf, len) + +#include "msgpack/pack_template.h" + +static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) +{ + pk->data = data; + pk->callback = callback; +} + +static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback) +{ + msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer)); + if(!pk) { return NULL; } + msgpack_packer_init(pk, data, callback); + return pk; +} + +static inline void msgpack_packer_free(msgpack_packer* pk) +{ + free(pk); +} + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/pack.h */ + diff --git a/python/unpack.h b/python/unpack.h new file mode 100644 index 0000000..d578360 --- /dev/null +++ b/python/unpack.h @@ -0,0 +1,397 @@ +/* + * MessagePack for Python unpacking routine + * + * Copyright (C) 2009 Naoki INADA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/unpack.h" +#include "msgpack/unpack_define.h" +#include + +#include "Python.h" + + +typedef struct { + int reserved; +} unpack_user; + + +#define msgpack_unpack_struct(name) \ + struct template ## name + +#define msgpack_unpack_func(ret, name) \ + ret template ## name + +#define msgpack_unpack_callback(name) \ + template_callback ## name + +#define msgpack_unpack_object PyObject* + +#define msgpack_unpack_user unpack_user + + +struct template_context; +typedef struct template_context template_context; + +static void template_init(template_context* ctx); + +static msgpack_unpack_object template_data(template_context* ctx); + +static int template_execute(template_context* ctx, + const char* data, size_t len, size_t* off); + + +static inline msgpack_unpack_object template_callback_root(unpack_user* u) +{ PyObject *o = Py_None; Py_INCREF(o); return o; } + +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong((long)d); return 0; } + +static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong((long)d); return 0; } + +static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) +{ + if (d >= 0x80000000UL) { + *o = PyLong_FromUnsignedLongLong((unsigned long long)d); + } else { + *o = PyInt_FromLong((long)d); + } + return 0; +} + +static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) +{ *o = PyLong_FromUnsignedLongLong(d); return 0; } + +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) +{ *o = PyLong_FromLongLong(d); return 0; } + +static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +{ *o = PyFloat_FromDouble((double)d); return 0; } + +static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) +{ *o = PyFloat_FromDouble(d); return 0; } + +static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) +{ *o = Py_None; Py_INCREF(o); return 0; } + +static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o) +{ *o = Py_True; Py_INCREF(o); return 0; } + +static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o) +{ *o = Py_False; Py_INCREF(o); return 0; } + +static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ + /* TODO: use PyList_New(n). */ + *o = PyList_New(0); + return 0; +} + +static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) +{ + PyList_Append(*c, o); + return 0; +} + +static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ + *o = PyDict_New(); + return 0; +} + +static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) +{ + PyDict_SetItem(*c, k, v); + return 0; +} + +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) +{ + *o = PyString_FromStringAndSize(p, l); + return 0; +} + +#include "msgpack/unpack_template.h" + + +#if 0 +#define CTX_CAST(m) ((template_context*)(m)) +#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced + + +static const size_t COUNTER_SIZE = sizeof(unsigned int); + +static inline void init_count(void* buffer) +{ + *(volatile unsigned int*)buffer = 1; +} + +static inline void decl_count(void* buffer) +{ + //if(--*(unsigned int*)buffer == 0) { + if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { + free(buffer); + } +} + +static inline void incr_count(void* buffer) +{ + //++*(unsigned int*)buffer; + __sync_add_and_fetch((unsigned int*)buffer, 1); +} + +static inline unsigned int get_count(void* buffer) +{ + return *(volatile unsigned int*)buffer; +} + + + +bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) +{ + if(initial_buffer_size < COUNTER_SIZE) { + initial_buffer_size = COUNTER_SIZE; + } + + char* buffer = (char*)malloc(initial_buffer_size); + if(buffer == NULL) { + return false; + } + + void* ctx = malloc(sizeof(template_context)); + if(ctx == NULL) { + free(buffer); + return false; + } + + msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(z == NULL) { + free(ctx); + free(buffer); + return false; + } + + mpac->buffer = buffer; + mpac->used = COUNTER_SIZE; + mpac->free = initial_buffer_size - mpac->used; + mpac->off = COUNTER_SIZE; + mpac->parsed = 0; + mpac->initial_buffer_size = initial_buffer_size; + mpac->z = z; + mpac->ctx = ctx; + + init_count(mpac->buffer); + + template_init(CTX_CAST(mpac->ctx)); + CTX_CAST(mpac->ctx)->user.z = mpac->z; + CTX_CAST(mpac->ctx)->user.referenced = false; + + return true; +} + +void msgpack_unpacker_destroy(msgpack_unpacker* mpac) +{ + msgpack_zone_free(mpac->z); + free(mpac->ctx); + decl_count(mpac->buffer); +} + + +msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size) +{ + msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker)); + if(mpac == NULL) { + return NULL; + } + + if(!msgpack_unpacker_init(mpac, initial_buffer_size)) { + free(mpac); + return NULL; + } + + return mpac; +} + +void msgpack_unpacker_free(msgpack_unpacker* mpac) +{ + msgpack_unpacker_destroy(mpac); + free(mpac); +} + +bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) +{ + if(mpac->used == mpac->off && get_count(mpac->buffer) == 1 + && !CTX_REFERENCED(mpac)) { + // rewind buffer + mpac->free += mpac->used - COUNTER_SIZE; + mpac->used = COUNTER_SIZE; + mpac->off = COUNTER_SIZE; + + if(mpac->free >= size) { + return true; + } + } + + if(mpac->off == COUNTER_SIZE) { + size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE + while(next_size < size + mpac->used) { + next_size *= 2; + } + + char* tmp = (char*)realloc(mpac->buffer, next_size); + if(tmp == NULL) { + return false; + } + + mpac->buffer = tmp; + mpac->free = next_size - mpac->used; + + } else { + size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE + size_t not_parsed = mpac->used - mpac->off; + while(next_size < size + not_parsed + COUNTER_SIZE) { + next_size *= 2; + } + + char* tmp = (char*)malloc(next_size); + if(tmp == NULL) { + return false; + } + + init_count(tmp); + + memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); + + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { + free(tmp); + return false; + } + CTX_REFERENCED(mpac) = false; + } else { + decl_count(mpac->buffer); + } + + mpac->buffer = tmp; + mpac->used = not_parsed + COUNTER_SIZE; + mpac->free = next_size - mpac->used; + mpac->off = COUNTER_SIZE; + } + + return true; +} + +int msgpack_unpacker_execute(msgpack_unpacker* mpac) +{ + size_t off = mpac->off; + int ret = template_execute(CTX_CAST(mpac->ctx), + mpac->buffer, mpac->used, &mpac->off); + if(mpac->off > off) { + mpac->parsed += mpac->off - off; + } + return ret; +} + +msgpack_unpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) +{ + return template_data(CTX_CAST(mpac->ctx)); +} + +msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) +{ + if(!msgpack_unpacker_flush_zone(mpac)) { + return false; + } + + msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(r == NULL) { + return NULL; + } + + msgpack_zone* old = mpac->z; + mpac->z = r; + + return old; +} + +void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac) +{ + msgpack_zone_clear(mpac->z); +} + +bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) +{ + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { + return false; + } + CTX_REFERENCED(mpac) = false; + + incr_count(mpac->buffer); + } + + return true; +} + +void msgpack_unpacker_reset(msgpack_unpacker* mpac) +{ + template_init(CTX_CAST(mpac->ctx)); + // don't reset referenced flag + mpac->parsed = 0; +} + + +msgpack_unpack_return +msgpack_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_unpack_object* result) +{ + template_context ctx; + template_init(&ctx); + + ctx.user.z = z; + ctx.user.referenced = false; + + size_t noff = 0; + if(off != NULL) { noff = *off; } + + int ret = template_execute(&ctx, data, len, &noff); + if(ret < 0) { + return MSGPACK_UNPACK_PARSE_ERROR; + } + + if(off != NULL) { *off = noff; } + + if(ret == 0) { + return MSGPACK_UNPACK_CONTINUE; + } + + *result = template_data(&ctx); + + if(noff < len) { + return MSGPACK_UNPACK_EXTRA_BYTES; + } + + return MSGPACK_UNPACK_SUCCESS; +} +#endif From 342d3ca71027b9a1f949fe03b5243557ee53dc94 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 00:23:38 +0900 Subject: [PATCH 0136/1172] support packing long and tuple. add missing files. --- msgpack.pyx | 53 ++++--- pack.h | 130 +++++++++++++++++ unpack.h | 397 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 563 insertions(+), 17 deletions(-) create mode 100644 pack.h create mode 100644 unpack.h diff --git a/msgpack.pyx b/msgpack.pyx index 6885317..ceae9b6 100644 --- a/msgpack.pyx +++ b/msgpack.pyx @@ -3,15 +3,17 @@ cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" + ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) cdef extern from "stdlib.h": void* malloc(int) void free(void*) + cdef extern from "string.h": int memcpy(char*dst, char*src, unsigned int size) -cdef extern from "msgpack/pack.h": +cdef extern from "pack.h": ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) struct msgpack_packer: @@ -30,7 +32,7 @@ cdef extern from "msgpack/pack.h": void msgpack_pack_raw(msgpack_packer* pk, size_t l) void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef extern from "msgpack/unpack.h": +cdef extern from "unpack.h": ctypedef struct msgpack_unpacker @@ -98,7 +100,7 @@ cdef class Packer: """ msgpack_pack_map(&self.pk, len) - def __call__(self, object o): + def pack(self, object o): cdef long long intval cdef double fval cdef char* rawval @@ -109,6 +111,9 @@ cdef class Packer: msgpack_pack_true(&self.pk) elif o is False: msgpack_pack_false(&self.pk) + elif isinstance(o, long): + intval = o + msgpack_pack_long_long(&self.pk, intval) elif isinstance(o, int): intval = o msgpack_pack_long_long(&self.pk, intval) @@ -120,24 +125,21 @@ cdef class Packer: msgpack_pack_raw(&self.pk, len(o)) msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif isinstance(o, unicode): - # todo - pass + o = o.encode('utf-8') + rawval = o + msgpack_pack_raw(&self.pk, len(o)) + msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif isinstance(o, dict): msgpack_pack_map(&self.pk, len(o)) for k,v in o.iteritems(): - self(k) - self(v) - elif isinstance(o, tuple): + self.pack(k) + self.pack(v) + elif isinstance(o, tuple) or isinstance(o, list): msgpack_pack_array(&self.pk, len(o)) for v in o: - self(v) - elif isinstance(o, list): - msgpack_pack_array(&self.pk, len(o)) - for v in o: - self(v) - elif hasattr(o, "__msgpack__"): - o.__msgpack__(self) + self.pack(v) else: + # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): @@ -155,8 +157,25 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): packer.length += l return 0 +cdef extern from "msgpack/zone.h": + ctypedef struct msgpack_zone + +cdef extern from "unpack.c": + ctypedef struct template_context: + pass + int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) + void template_init(template_context* ctx) + PyObject* template_data(template_context* ctx) + + cdef class Unpacker: def __init__(self): pass - def unpack(strm): - pass + + def unpack(self, bytes_): + cdef const_char_ptr p = bytes_ + cdef template_context ctx + cdef size_t off = 0 + template_init(&ctx) + template_execute(&ctx, p, len(bytes_), &off) + return template_data(&ctx) diff --git a/pack.h b/pack.h new file mode 100644 index 0000000..4a336d5 --- /dev/null +++ b/pack.h @@ -0,0 +1,130 @@ +/* + * MessagePack for Python packing routine + * + * Copyright (C) 2009 Naoki INADA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_H__ +#define MSGPACK_PACK_H__ + +#if _MSC_VER +typedef signed char uint8_t; +typedef unsigned char uint8_t; +typedef short int16_t; +typedef unsigned short uint16_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef long long int64_t; +typedef unsigned long long uint64_t; +#elif +#include +#endif + +#include +#include +#include "msgpack/pack_define.h" +#include "msgpack/object.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len); + +typedef struct msgpack_packer { + void* data; + msgpack_packer_write callback; +} msgpack_packer; + +static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); + +static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); +static void msgpack_packer_free(msgpack_packer* pk); + +static int msgpack_pack_short(msgpack_packer* pk, short d); +static int msgpack_pack_int(msgpack_packer* pk, int d); +static int msgpack_pack_long(msgpack_packer* pk, long d); +static int msgpack_pack_long_long(msgpack_packer* pk, long long d); +static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); +static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); +static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); +static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); + +static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); +static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); +static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); +static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); +static int msgpack_pack_int8(msgpack_packer* pk, int8_t d); +static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); +static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); +static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); + +static int msgpack_pack_float(msgpack_packer* pk, float d); +static int msgpack_pack_double(msgpack_packer* pk, double d); + +static int msgpack_pack_nil(msgpack_packer* pk); +static int msgpack_pack_true(msgpack_packer* pk); +static int msgpack_pack_false(msgpack_packer* pk); + +static int msgpack_pack_array(msgpack_packer* pk, unsigned int n); + +static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); + +static int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); + +int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); + + + +#define msgpack_pack_inline_func(name) \ + static inline int msgpack_pack ## name + +#define msgpack_pack_inline_func_cint(name) \ + static inline int msgpack_pack ## name + +#define msgpack_pack_user msgpack_packer* + +#define msgpack_pack_append_buffer(user, buf, len) \ + return (*(user)->callback)((user)->data, (const char*)buf, len) + +#include "msgpack/pack_template.h" + +static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) +{ + pk->data = data; + pk->callback = callback; +} + +static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback) +{ + msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer)); + if(!pk) { return NULL; } + msgpack_packer_init(pk, data, callback); + return pk; +} + +static inline void msgpack_packer_free(msgpack_packer* pk) +{ + free(pk); +} + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/pack.h */ + diff --git a/unpack.h b/unpack.h new file mode 100644 index 0000000..d578360 --- /dev/null +++ b/unpack.h @@ -0,0 +1,397 @@ +/* + * MessagePack for Python unpacking routine + * + * Copyright (C) 2009 Naoki INADA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/unpack.h" +#include "msgpack/unpack_define.h" +#include + +#include "Python.h" + + +typedef struct { + int reserved; +} unpack_user; + + +#define msgpack_unpack_struct(name) \ + struct template ## name + +#define msgpack_unpack_func(ret, name) \ + ret template ## name + +#define msgpack_unpack_callback(name) \ + template_callback ## name + +#define msgpack_unpack_object PyObject* + +#define msgpack_unpack_user unpack_user + + +struct template_context; +typedef struct template_context template_context; + +static void template_init(template_context* ctx); + +static msgpack_unpack_object template_data(template_context* ctx); + +static int template_execute(template_context* ctx, + const char* data, size_t len, size_t* off); + + +static inline msgpack_unpack_object template_callback_root(unpack_user* u) +{ PyObject *o = Py_None; Py_INCREF(o); return o; } + +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong((long)d); return 0; } + +static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong((long)d); return 0; } + +static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) +{ + if (d >= 0x80000000UL) { + *o = PyLong_FromUnsignedLongLong((unsigned long long)d); + } else { + *o = PyInt_FromLong((long)d); + } + return 0; +} + +static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) +{ *o = PyLong_FromUnsignedLongLong(d); return 0; } + +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) +{ *o = PyLong_FromLongLong(d); return 0; } + +static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +{ *o = PyFloat_FromDouble((double)d); return 0; } + +static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) +{ *o = PyFloat_FromDouble(d); return 0; } + +static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) +{ *o = Py_None; Py_INCREF(o); return 0; } + +static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o) +{ *o = Py_True; Py_INCREF(o); return 0; } + +static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o) +{ *o = Py_False; Py_INCREF(o); return 0; } + +static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ + /* TODO: use PyList_New(n). */ + *o = PyList_New(0); + return 0; +} + +static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) +{ + PyList_Append(*c, o); + return 0; +} + +static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ + *o = PyDict_New(); + return 0; +} + +static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) +{ + PyDict_SetItem(*c, k, v); + return 0; +} + +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) +{ + *o = PyString_FromStringAndSize(p, l); + return 0; +} + +#include "msgpack/unpack_template.h" + + +#if 0 +#define CTX_CAST(m) ((template_context*)(m)) +#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced + + +static const size_t COUNTER_SIZE = sizeof(unsigned int); + +static inline void init_count(void* buffer) +{ + *(volatile unsigned int*)buffer = 1; +} + +static inline void decl_count(void* buffer) +{ + //if(--*(unsigned int*)buffer == 0) { + if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { + free(buffer); + } +} + +static inline void incr_count(void* buffer) +{ + //++*(unsigned int*)buffer; + __sync_add_and_fetch((unsigned int*)buffer, 1); +} + +static inline unsigned int get_count(void* buffer) +{ + return *(volatile unsigned int*)buffer; +} + + + +bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) +{ + if(initial_buffer_size < COUNTER_SIZE) { + initial_buffer_size = COUNTER_SIZE; + } + + char* buffer = (char*)malloc(initial_buffer_size); + if(buffer == NULL) { + return false; + } + + void* ctx = malloc(sizeof(template_context)); + if(ctx == NULL) { + free(buffer); + return false; + } + + msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(z == NULL) { + free(ctx); + free(buffer); + return false; + } + + mpac->buffer = buffer; + mpac->used = COUNTER_SIZE; + mpac->free = initial_buffer_size - mpac->used; + mpac->off = COUNTER_SIZE; + mpac->parsed = 0; + mpac->initial_buffer_size = initial_buffer_size; + mpac->z = z; + mpac->ctx = ctx; + + init_count(mpac->buffer); + + template_init(CTX_CAST(mpac->ctx)); + CTX_CAST(mpac->ctx)->user.z = mpac->z; + CTX_CAST(mpac->ctx)->user.referenced = false; + + return true; +} + +void msgpack_unpacker_destroy(msgpack_unpacker* mpac) +{ + msgpack_zone_free(mpac->z); + free(mpac->ctx); + decl_count(mpac->buffer); +} + + +msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size) +{ + msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker)); + if(mpac == NULL) { + return NULL; + } + + if(!msgpack_unpacker_init(mpac, initial_buffer_size)) { + free(mpac); + return NULL; + } + + return mpac; +} + +void msgpack_unpacker_free(msgpack_unpacker* mpac) +{ + msgpack_unpacker_destroy(mpac); + free(mpac); +} + +bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) +{ + if(mpac->used == mpac->off && get_count(mpac->buffer) == 1 + && !CTX_REFERENCED(mpac)) { + // rewind buffer + mpac->free += mpac->used - COUNTER_SIZE; + mpac->used = COUNTER_SIZE; + mpac->off = COUNTER_SIZE; + + if(mpac->free >= size) { + return true; + } + } + + if(mpac->off == COUNTER_SIZE) { + size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE + while(next_size < size + mpac->used) { + next_size *= 2; + } + + char* tmp = (char*)realloc(mpac->buffer, next_size); + if(tmp == NULL) { + return false; + } + + mpac->buffer = tmp; + mpac->free = next_size - mpac->used; + + } else { + size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE + size_t not_parsed = mpac->used - mpac->off; + while(next_size < size + not_parsed + COUNTER_SIZE) { + next_size *= 2; + } + + char* tmp = (char*)malloc(next_size); + if(tmp == NULL) { + return false; + } + + init_count(tmp); + + memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); + + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { + free(tmp); + return false; + } + CTX_REFERENCED(mpac) = false; + } else { + decl_count(mpac->buffer); + } + + mpac->buffer = tmp; + mpac->used = not_parsed + COUNTER_SIZE; + mpac->free = next_size - mpac->used; + mpac->off = COUNTER_SIZE; + } + + return true; +} + +int msgpack_unpacker_execute(msgpack_unpacker* mpac) +{ + size_t off = mpac->off; + int ret = template_execute(CTX_CAST(mpac->ctx), + mpac->buffer, mpac->used, &mpac->off); + if(mpac->off > off) { + mpac->parsed += mpac->off - off; + } + return ret; +} + +msgpack_unpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) +{ + return template_data(CTX_CAST(mpac->ctx)); +} + +msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) +{ + if(!msgpack_unpacker_flush_zone(mpac)) { + return false; + } + + msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(r == NULL) { + return NULL; + } + + msgpack_zone* old = mpac->z; + mpac->z = r; + + return old; +} + +void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac) +{ + msgpack_zone_clear(mpac->z); +} + +bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) +{ + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { + return false; + } + CTX_REFERENCED(mpac) = false; + + incr_count(mpac->buffer); + } + + return true; +} + +void msgpack_unpacker_reset(msgpack_unpacker* mpac) +{ + template_init(CTX_CAST(mpac->ctx)); + // don't reset referenced flag + mpac->parsed = 0; +} + + +msgpack_unpack_return +msgpack_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_unpack_object* result) +{ + template_context ctx; + template_init(&ctx); + + ctx.user.z = z; + ctx.user.referenced = false; + + size_t noff = 0; + if(off != NULL) { noff = *off; } + + int ret = template_execute(&ctx, data, len, &noff); + if(ret < 0) { + return MSGPACK_UNPACK_PARSE_ERROR; + } + + if(off != NULL) { *off = noff; } + + if(ret == 0) { + return MSGPACK_UNPACK_CONTINUE; + } + + *result = template_data(&ctx); + + if(noff < len) { + return MSGPACK_UNPACK_EXTRA_BYTES; + } + + return MSGPACK_UNPACK_SUCCESS; +} +#endif From d8a3bc920caa06f21adcfc288c4c597bb9d33157 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 01:30:43 +0900 Subject: [PATCH 0137/1172] refactoring --- python/msgpack.pyx | 58 ++++++---- python/pack.h | 67 +++++------ python/unpack.h | 273 +-------------------------------------------- 3 files changed, 68 insertions(+), 330 deletions(-) diff --git a/python/msgpack.pyx b/python/msgpack.pyx index ceae9b6..c454c5d 100644 --- a/python/msgpack.pyx +++ b/python/msgpack.pyx @@ -1,5 +1,6 @@ # coding: utf-8 +from cStringIO import StringIO cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" @@ -32,13 +33,15 @@ cdef extern from "pack.h": void msgpack_pack_raw(msgpack_packer* pk, size_t l) void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef extern from "unpack.h": - ctypedef struct msgpack_unpacker - cdef int BUFF_SIZE=2*1024 cdef class Packer: + """Packer that pack data into strm. + + strm must have `write(bytes)` method. + size specifies local buffer size. + """ cdef char* buff cdef unsigned int length cdef unsigned int allocated @@ -46,11 +49,6 @@ cdef class Packer: cdef object strm def __init__(self, strm, int size=0): - """Make packer that pack data into strm. - - strm must have `write(bytes)` method. - size specifies local buffer size. - """ if size <= 0: size = BUFF_SIZE @@ -157,25 +155,41 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): packer.length += l return 0 -cdef extern from "msgpack/zone.h": - ctypedef struct msgpack_zone +def pack(object o, object stream): + packer = Packer(stream) + packer.pack(o) + packer.flush() -cdef extern from "unpack.c": +def packs(object o): + buf = StringIO() + packer = Packer(buf) + packer.pack(o) + packer.flush() + return buf.getvalue() + +cdef extern from "unpack.h": ctypedef struct template_context: pass - int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) + int template_execute(template_context* ctx, const_char_ptr data, + size_t len, size_t* off) void template_init(template_context* ctx) PyObject* template_data(template_context* ctx) -cdef class Unpacker: - def __init__(self): - pass +def unpacks(object packed_bytes): + """Unpack packed_bytes to object. Returns unpacked object.""" + cdef const_char_ptr p = packed_bytes + cdef template_context ctx + cdef size_t off = 0 + template_init(&ctx) + template_execute(&ctx, p, len(packed_bytes), &off) + return template_data(&ctx) - def unpack(self, bytes_): - cdef const_char_ptr p = bytes_ - cdef template_context ctx - cdef size_t off = 0 - template_init(&ctx) - template_execute(&ctx, p, len(bytes_), &off) - return template_data(&ctx) +def unpack(object stream): + """unpack from stream.""" + packed = stream.read() + return unpacks(packed) + +cdef class Unpacker: + """Do nothing. This function is for symmetric to Packer""" + unpack = staticmethod(unpacks) diff --git a/python/pack.h b/python/pack.h index 4a336d5..f3935fb 100644 --- a/python/pack.h +++ b/python/pack.h @@ -15,9 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MSGPACK_PACK_H__ -#define MSGPACK_PACK_H__ - #if _MSC_VER typedef signed char uint8_t; typedef unsigned char uint8_t; @@ -27,14 +24,13 @@ typedef int int32_t; typedef unsigned int uint32_t; typedef long long int64_t; typedef unsigned long long uint64_t; -#elif +#else #include #endif #include #include #include "msgpack/pack_define.h" -#include "msgpack/object.h" #ifdef __cplusplus extern "C" { @@ -48,44 +44,42 @@ typedef struct msgpack_packer { msgpack_packer_write callback; } msgpack_packer; -static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); +static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); -static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); -static void msgpack_packer_free(msgpack_packer* pk); +static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); +static inline void msgpack_packer_free(msgpack_packer* pk); -static int msgpack_pack_short(msgpack_packer* pk, short d); -static int msgpack_pack_int(msgpack_packer* pk, int d); -static int msgpack_pack_long(msgpack_packer* pk, long d); -static int msgpack_pack_long_long(msgpack_packer* pk, long long d); -static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); -static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); -static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); -static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); +static inline int msgpack_pack_short(msgpack_packer* pk, short d); +static inline int msgpack_pack_int(msgpack_packer* pk, int d); +static inline int msgpack_pack_long(msgpack_packer* pk, long d); +static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d); +static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); +static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); +static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); +static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); -static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); -static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); -static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); -static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); -static int msgpack_pack_int8(msgpack_packer* pk, int8_t d); -static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); -static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); -static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); +static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); +static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); +static inline int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); +static inline int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); +static inline int msgpack_pack_int8(msgpack_packer* pk, int8_t d); +static inline int msgpack_pack_int16(msgpack_packer* pk, int16_t d); +static inline int msgpack_pack_int32(msgpack_packer* pk, int32_t d); +static inline int msgpack_pack_int64(msgpack_packer* pk, int64_t d); -static int msgpack_pack_float(msgpack_packer* pk, float d); -static int msgpack_pack_double(msgpack_packer* pk, double d); +static inline int msgpack_pack_float(msgpack_packer* pk, float d); +static inline int msgpack_pack_double(msgpack_packer* pk, double d); -static int msgpack_pack_nil(msgpack_packer* pk); -static int msgpack_pack_true(msgpack_packer* pk); -static int msgpack_pack_false(msgpack_packer* pk); +static inline int msgpack_pack_nil(msgpack_packer* pk); +static inline int msgpack_pack_true(msgpack_packer* pk); +static inline int msgpack_pack_false(msgpack_packer* pk); -static int msgpack_pack_array(msgpack_packer* pk, unsigned int n); +static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); +static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_raw(msgpack_packer* pk, size_t l); -static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); - -int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); +static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); @@ -125,6 +119,3 @@ static inline void msgpack_packer_free(msgpack_packer* pk) #ifdef __cplusplus } #endif - -#endif /* msgpack/pack.h */ - diff --git a/python/unpack.h b/python/unpack.h index d578360..eb72001 100644 --- a/python/unpack.h +++ b/python/unpack.h @@ -15,12 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "msgpack/unpack.h" #include "msgpack/unpack_define.h" -#include - -#include "Python.h" - typedef struct { int reserved; @@ -44,11 +39,11 @@ typedef struct { struct template_context; typedef struct template_context template_context; -static void template_init(template_context* ctx); +static inline void template_init(template_context* ctx); -static msgpack_unpack_object template_data(template_context* ctx); +static inline msgpack_unpack_object template_data(template_context* ctx); -static int template_execute(template_context* ctx, +static inline int template_execute(template_context* ctx, const char* data, size_t len, size_t* off); @@ -133,265 +128,3 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha } #include "msgpack/unpack_template.h" - - -#if 0 -#define CTX_CAST(m) ((template_context*)(m)) -#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced - - -static const size_t COUNTER_SIZE = sizeof(unsigned int); - -static inline void init_count(void* buffer) -{ - *(volatile unsigned int*)buffer = 1; -} - -static inline void decl_count(void* buffer) -{ - //if(--*(unsigned int*)buffer == 0) { - if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { - free(buffer); - } -} - -static inline void incr_count(void* buffer) -{ - //++*(unsigned int*)buffer; - __sync_add_and_fetch((unsigned int*)buffer, 1); -} - -static inline unsigned int get_count(void* buffer) -{ - return *(volatile unsigned int*)buffer; -} - - - -bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) -{ - if(initial_buffer_size < COUNTER_SIZE) { - initial_buffer_size = COUNTER_SIZE; - } - - char* buffer = (char*)malloc(initial_buffer_size); - if(buffer == NULL) { - return false; - } - - void* ctx = malloc(sizeof(template_context)); - if(ctx == NULL) { - free(buffer); - return false; - } - - msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); - if(z == NULL) { - free(ctx); - free(buffer); - return false; - } - - mpac->buffer = buffer; - mpac->used = COUNTER_SIZE; - mpac->free = initial_buffer_size - mpac->used; - mpac->off = COUNTER_SIZE; - mpac->parsed = 0; - mpac->initial_buffer_size = initial_buffer_size; - mpac->z = z; - mpac->ctx = ctx; - - init_count(mpac->buffer); - - template_init(CTX_CAST(mpac->ctx)); - CTX_CAST(mpac->ctx)->user.z = mpac->z; - CTX_CAST(mpac->ctx)->user.referenced = false; - - return true; -} - -void msgpack_unpacker_destroy(msgpack_unpacker* mpac) -{ - msgpack_zone_free(mpac->z); - free(mpac->ctx); - decl_count(mpac->buffer); -} - - -msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size) -{ - msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker)); - if(mpac == NULL) { - return NULL; - } - - if(!msgpack_unpacker_init(mpac, initial_buffer_size)) { - free(mpac); - return NULL; - } - - return mpac; -} - -void msgpack_unpacker_free(msgpack_unpacker* mpac) -{ - msgpack_unpacker_destroy(mpac); - free(mpac); -} - -bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) -{ - if(mpac->used == mpac->off && get_count(mpac->buffer) == 1 - && !CTX_REFERENCED(mpac)) { - // rewind buffer - mpac->free += mpac->used - COUNTER_SIZE; - mpac->used = COUNTER_SIZE; - mpac->off = COUNTER_SIZE; - - if(mpac->free >= size) { - return true; - } - } - - if(mpac->off == COUNTER_SIZE) { - size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE - while(next_size < size + mpac->used) { - next_size *= 2; - } - - char* tmp = (char*)realloc(mpac->buffer, next_size); - if(tmp == NULL) { - return false; - } - - mpac->buffer = tmp; - mpac->free = next_size - mpac->used; - - } else { - size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE - size_t not_parsed = mpac->used - mpac->off; - while(next_size < size + not_parsed + COUNTER_SIZE) { - next_size *= 2; - } - - char* tmp = (char*)malloc(next_size); - if(tmp == NULL) { - return false; - } - - init_count(tmp); - - memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); - - if(CTX_REFERENCED(mpac)) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { - free(tmp); - return false; - } - CTX_REFERENCED(mpac) = false; - } else { - decl_count(mpac->buffer); - } - - mpac->buffer = tmp; - mpac->used = not_parsed + COUNTER_SIZE; - mpac->free = next_size - mpac->used; - mpac->off = COUNTER_SIZE; - } - - return true; -} - -int msgpack_unpacker_execute(msgpack_unpacker* mpac) -{ - size_t off = mpac->off; - int ret = template_execute(CTX_CAST(mpac->ctx), - mpac->buffer, mpac->used, &mpac->off); - if(mpac->off > off) { - mpac->parsed += mpac->off - off; - } - return ret; -} - -msgpack_unpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) -{ - return template_data(CTX_CAST(mpac->ctx)); -} - -msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) -{ - if(!msgpack_unpacker_flush_zone(mpac)) { - return false; - } - - msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); - if(r == NULL) { - return NULL; - } - - msgpack_zone* old = mpac->z; - mpac->z = r; - - return old; -} - -void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac) -{ - msgpack_zone_clear(mpac->z); -} - -bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) -{ - if(CTX_REFERENCED(mpac)) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { - return false; - } - CTX_REFERENCED(mpac) = false; - - incr_count(mpac->buffer); - } - - return true; -} - -void msgpack_unpacker_reset(msgpack_unpacker* mpac) -{ - template_init(CTX_CAST(mpac->ctx)); - // don't reset referenced flag - mpac->parsed = 0; -} - - -msgpack_unpack_return -msgpack_unpack(const char* data, size_t len, size_t* off, - msgpack_zone* z, msgpack_unpack_object* result) -{ - template_context ctx; - template_init(&ctx); - - ctx.user.z = z; - ctx.user.referenced = false; - - size_t noff = 0; - if(off != NULL) { noff = *off; } - - int ret = template_execute(&ctx, data, len, &noff); - if(ret < 0) { - return MSGPACK_UNPACK_PARSE_ERROR; - } - - if(off != NULL) { *off = noff; } - - if(ret == 0) { - return MSGPACK_UNPACK_CONTINUE; - } - - *result = template_data(&ctx); - - if(noff < len) { - return MSGPACK_UNPACK_EXTRA_BYTES; - } - - return MSGPACK_UNPACK_SUCCESS; -} -#endif From 5425b72247ca4f0de9e6780fb4405acaef5ae986 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 01:30:43 +0900 Subject: [PATCH 0138/1172] refactoring --- msgpack.pyx | 58 ++++++----- pack.h | 67 ++++++------- unpack.h | 273 +--------------------------------------------------- 3 files changed, 68 insertions(+), 330 deletions(-) diff --git a/msgpack.pyx b/msgpack.pyx index ceae9b6..c454c5d 100644 --- a/msgpack.pyx +++ b/msgpack.pyx @@ -1,5 +1,6 @@ # coding: utf-8 +from cStringIO import StringIO cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" @@ -32,13 +33,15 @@ cdef extern from "pack.h": void msgpack_pack_raw(msgpack_packer* pk, size_t l) void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef extern from "unpack.h": - ctypedef struct msgpack_unpacker - cdef int BUFF_SIZE=2*1024 cdef class Packer: + """Packer that pack data into strm. + + strm must have `write(bytes)` method. + size specifies local buffer size. + """ cdef char* buff cdef unsigned int length cdef unsigned int allocated @@ -46,11 +49,6 @@ cdef class Packer: cdef object strm def __init__(self, strm, int size=0): - """Make packer that pack data into strm. - - strm must have `write(bytes)` method. - size specifies local buffer size. - """ if size <= 0: size = BUFF_SIZE @@ -157,25 +155,41 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): packer.length += l return 0 -cdef extern from "msgpack/zone.h": - ctypedef struct msgpack_zone +def pack(object o, object stream): + packer = Packer(stream) + packer.pack(o) + packer.flush() -cdef extern from "unpack.c": +def packs(object o): + buf = StringIO() + packer = Packer(buf) + packer.pack(o) + packer.flush() + return buf.getvalue() + +cdef extern from "unpack.h": ctypedef struct template_context: pass - int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) + int template_execute(template_context* ctx, const_char_ptr data, + size_t len, size_t* off) void template_init(template_context* ctx) PyObject* template_data(template_context* ctx) -cdef class Unpacker: - def __init__(self): - pass +def unpacks(object packed_bytes): + """Unpack packed_bytes to object. Returns unpacked object.""" + cdef const_char_ptr p = packed_bytes + cdef template_context ctx + cdef size_t off = 0 + template_init(&ctx) + template_execute(&ctx, p, len(packed_bytes), &off) + return template_data(&ctx) - def unpack(self, bytes_): - cdef const_char_ptr p = bytes_ - cdef template_context ctx - cdef size_t off = 0 - template_init(&ctx) - template_execute(&ctx, p, len(bytes_), &off) - return template_data(&ctx) +def unpack(object stream): + """unpack from stream.""" + packed = stream.read() + return unpacks(packed) + +cdef class Unpacker: + """Do nothing. This function is for symmetric to Packer""" + unpack = staticmethod(unpacks) diff --git a/pack.h b/pack.h index 4a336d5..f3935fb 100644 --- a/pack.h +++ b/pack.h @@ -15,9 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MSGPACK_PACK_H__ -#define MSGPACK_PACK_H__ - #if _MSC_VER typedef signed char uint8_t; typedef unsigned char uint8_t; @@ -27,14 +24,13 @@ typedef int int32_t; typedef unsigned int uint32_t; typedef long long int64_t; typedef unsigned long long uint64_t; -#elif +#else #include #endif #include #include #include "msgpack/pack_define.h" -#include "msgpack/object.h" #ifdef __cplusplus extern "C" { @@ -48,44 +44,42 @@ typedef struct msgpack_packer { msgpack_packer_write callback; } msgpack_packer; -static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); +static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); -static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); -static void msgpack_packer_free(msgpack_packer* pk); +static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); +static inline void msgpack_packer_free(msgpack_packer* pk); -static int msgpack_pack_short(msgpack_packer* pk, short d); -static int msgpack_pack_int(msgpack_packer* pk, int d); -static int msgpack_pack_long(msgpack_packer* pk, long d); -static int msgpack_pack_long_long(msgpack_packer* pk, long long d); -static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); -static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); -static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); -static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); +static inline int msgpack_pack_short(msgpack_packer* pk, short d); +static inline int msgpack_pack_int(msgpack_packer* pk, int d); +static inline int msgpack_pack_long(msgpack_packer* pk, long d); +static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d); +static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); +static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); +static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); +static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); -static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); -static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); -static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); -static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); -static int msgpack_pack_int8(msgpack_packer* pk, int8_t d); -static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); -static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); -static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); +static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); +static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); +static inline int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); +static inline int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); +static inline int msgpack_pack_int8(msgpack_packer* pk, int8_t d); +static inline int msgpack_pack_int16(msgpack_packer* pk, int16_t d); +static inline int msgpack_pack_int32(msgpack_packer* pk, int32_t d); +static inline int msgpack_pack_int64(msgpack_packer* pk, int64_t d); -static int msgpack_pack_float(msgpack_packer* pk, float d); -static int msgpack_pack_double(msgpack_packer* pk, double d); +static inline int msgpack_pack_float(msgpack_packer* pk, float d); +static inline int msgpack_pack_double(msgpack_packer* pk, double d); -static int msgpack_pack_nil(msgpack_packer* pk); -static int msgpack_pack_true(msgpack_packer* pk); -static int msgpack_pack_false(msgpack_packer* pk); +static inline int msgpack_pack_nil(msgpack_packer* pk); +static inline int msgpack_pack_true(msgpack_packer* pk); +static inline int msgpack_pack_false(msgpack_packer* pk); -static int msgpack_pack_array(msgpack_packer* pk, unsigned int n); +static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); +static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_raw(msgpack_packer* pk, size_t l); -static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); - -int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); +static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); @@ -125,6 +119,3 @@ static inline void msgpack_packer_free(msgpack_packer* pk) #ifdef __cplusplus } #endif - -#endif /* msgpack/pack.h */ - diff --git a/unpack.h b/unpack.h index d578360..eb72001 100644 --- a/unpack.h +++ b/unpack.h @@ -15,12 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "msgpack/unpack.h" #include "msgpack/unpack_define.h" -#include - -#include "Python.h" - typedef struct { int reserved; @@ -44,11 +39,11 @@ typedef struct { struct template_context; typedef struct template_context template_context; -static void template_init(template_context* ctx); +static inline void template_init(template_context* ctx); -static msgpack_unpack_object template_data(template_context* ctx); +static inline msgpack_unpack_object template_data(template_context* ctx); -static int template_execute(template_context* ctx, +static inline int template_execute(template_context* ctx, const char* data, size_t len, size_t* off); @@ -133,265 +128,3 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha } #include "msgpack/unpack_template.h" - - -#if 0 -#define CTX_CAST(m) ((template_context*)(m)) -#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced - - -static const size_t COUNTER_SIZE = sizeof(unsigned int); - -static inline void init_count(void* buffer) -{ - *(volatile unsigned int*)buffer = 1; -} - -static inline void decl_count(void* buffer) -{ - //if(--*(unsigned int*)buffer == 0) { - if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { - free(buffer); - } -} - -static inline void incr_count(void* buffer) -{ - //++*(unsigned int*)buffer; - __sync_add_and_fetch((unsigned int*)buffer, 1); -} - -static inline unsigned int get_count(void* buffer) -{ - return *(volatile unsigned int*)buffer; -} - - - -bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) -{ - if(initial_buffer_size < COUNTER_SIZE) { - initial_buffer_size = COUNTER_SIZE; - } - - char* buffer = (char*)malloc(initial_buffer_size); - if(buffer == NULL) { - return false; - } - - void* ctx = malloc(sizeof(template_context)); - if(ctx == NULL) { - free(buffer); - return false; - } - - msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); - if(z == NULL) { - free(ctx); - free(buffer); - return false; - } - - mpac->buffer = buffer; - mpac->used = COUNTER_SIZE; - mpac->free = initial_buffer_size - mpac->used; - mpac->off = COUNTER_SIZE; - mpac->parsed = 0; - mpac->initial_buffer_size = initial_buffer_size; - mpac->z = z; - mpac->ctx = ctx; - - init_count(mpac->buffer); - - template_init(CTX_CAST(mpac->ctx)); - CTX_CAST(mpac->ctx)->user.z = mpac->z; - CTX_CAST(mpac->ctx)->user.referenced = false; - - return true; -} - -void msgpack_unpacker_destroy(msgpack_unpacker* mpac) -{ - msgpack_zone_free(mpac->z); - free(mpac->ctx); - decl_count(mpac->buffer); -} - - -msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size) -{ - msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker)); - if(mpac == NULL) { - return NULL; - } - - if(!msgpack_unpacker_init(mpac, initial_buffer_size)) { - free(mpac); - return NULL; - } - - return mpac; -} - -void msgpack_unpacker_free(msgpack_unpacker* mpac) -{ - msgpack_unpacker_destroy(mpac); - free(mpac); -} - -bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) -{ - if(mpac->used == mpac->off && get_count(mpac->buffer) == 1 - && !CTX_REFERENCED(mpac)) { - // rewind buffer - mpac->free += mpac->used - COUNTER_SIZE; - mpac->used = COUNTER_SIZE; - mpac->off = COUNTER_SIZE; - - if(mpac->free >= size) { - return true; - } - } - - if(mpac->off == COUNTER_SIZE) { - size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE - while(next_size < size + mpac->used) { - next_size *= 2; - } - - char* tmp = (char*)realloc(mpac->buffer, next_size); - if(tmp == NULL) { - return false; - } - - mpac->buffer = tmp; - mpac->free = next_size - mpac->used; - - } else { - size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE - size_t not_parsed = mpac->used - mpac->off; - while(next_size < size + not_parsed + COUNTER_SIZE) { - next_size *= 2; - } - - char* tmp = (char*)malloc(next_size); - if(tmp == NULL) { - return false; - } - - init_count(tmp); - - memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); - - if(CTX_REFERENCED(mpac)) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { - free(tmp); - return false; - } - CTX_REFERENCED(mpac) = false; - } else { - decl_count(mpac->buffer); - } - - mpac->buffer = tmp; - mpac->used = not_parsed + COUNTER_SIZE; - mpac->free = next_size - mpac->used; - mpac->off = COUNTER_SIZE; - } - - return true; -} - -int msgpack_unpacker_execute(msgpack_unpacker* mpac) -{ - size_t off = mpac->off; - int ret = template_execute(CTX_CAST(mpac->ctx), - mpac->buffer, mpac->used, &mpac->off); - if(mpac->off > off) { - mpac->parsed += mpac->off - off; - } - return ret; -} - -msgpack_unpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) -{ - return template_data(CTX_CAST(mpac->ctx)); -} - -msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) -{ - if(!msgpack_unpacker_flush_zone(mpac)) { - return false; - } - - msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); - if(r == NULL) { - return NULL; - } - - msgpack_zone* old = mpac->z; - mpac->z = r; - - return old; -} - -void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac) -{ - msgpack_zone_clear(mpac->z); -} - -bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) -{ - if(CTX_REFERENCED(mpac)) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { - return false; - } - CTX_REFERENCED(mpac) = false; - - incr_count(mpac->buffer); - } - - return true; -} - -void msgpack_unpacker_reset(msgpack_unpacker* mpac) -{ - template_init(CTX_CAST(mpac->ctx)); - // don't reset referenced flag - mpac->parsed = 0; -} - - -msgpack_unpack_return -msgpack_unpack(const char* data, size_t len, size_t* off, - msgpack_zone* z, msgpack_unpack_object* result) -{ - template_context ctx; - template_init(&ctx); - - ctx.user.z = z; - ctx.user.referenced = false; - - size_t noff = 0; - if(off != NULL) { noff = *off; } - - int ret = template_execute(&ctx, data, len, &noff); - if(ret < 0) { - return MSGPACK_UNPACK_PARSE_ERROR; - } - - if(off != NULL) { *off = noff; } - - if(ret == 0) { - return MSGPACK_UNPACK_CONTINUE; - } - - *result = template_data(&ctx); - - if(noff < len) { - return MSGPACK_UNPACK_EXTRA_BYTES; - } - - return MSGPACK_UNPACK_SUCCESS; -} -#endif From c930f5367b41ef141e80a76506fd91b49fe7d426 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 01:43:50 +0900 Subject: [PATCH 0139/1172] add cythoned source and setup script. --- python/msgpack | 1 + python/msgpack.c | 3231 ++++++++++++++++++++++++++++++++++++++++++++++ python/setup.py | 29 + 3 files changed, 3261 insertions(+) create mode 120000 python/msgpack create mode 100644 python/msgpack.c create mode 100644 python/setup.py diff --git a/python/msgpack b/python/msgpack new file mode 120000 index 0000000..430db49 --- /dev/null +++ b/python/msgpack @@ -0,0 +1 @@ +../msgpack \ No newline at end of file diff --git a/python/msgpack.c b/python/msgpack.c new file mode 100644 index 0000000..50ec6fc --- /dev/null +++ b/python/msgpack.c @@ -0,0 +1,3231 @@ +/* Generated by Cython 0.11.2 on Mon Jun 8 01:28:30 2009 */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include "structmember.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#if PY_VERSION_HEX < 0x02040000 + #define METH_COEXIST 0 + #define PyDict_CheckExact(op) (Py_TYPE(op) == &PyDict_Type) +#endif +#if PY_VERSION_HEX < 0x02050000 + typedef int Py_ssize_t; + #define PY_SSIZE_T_MAX INT_MAX + #define PY_SSIZE_T_MIN INT_MIN + #define PY_FORMAT_SIZE_T "" + #define PyInt_FromSsize_t(z) PyInt_FromLong(z) + #define PyInt_AsSsize_t(o) PyInt_AsLong(o) + #define PyNumber_Index(o) PyNumber_Int(o) + #define PyIndex_Check(o) PyNumber_Check(o) +#endif +#if PY_VERSION_HEX < 0x02060000 + #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) + #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) + #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) + #define PyVarObject_HEAD_INIT(type, size) \ + PyObject_HEAD_INIT(type) size, + #define PyType_Modified(t) + + typedef struct { + void *buf; + PyObject *obj; + Py_ssize_t len; + Py_ssize_t itemsize; + int readonly; + int ndim; + char *format; + Py_ssize_t *shape; + Py_ssize_t *strides; + Py_ssize_t *suboffsets; + void *internal; + } Py_buffer; + + #define PyBUF_SIMPLE 0 + #define PyBUF_WRITABLE 0x0001 + #define PyBUF_FORMAT 0x0004 + #define PyBUF_ND 0x0008 + #define PyBUF_STRIDES (0x0010 | PyBUF_ND) + #define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES) + #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES) + #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES) + #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES) + +#endif +#if PY_MAJOR_VERSION < 3 + #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" +#else + #define __Pyx_BUILTIN_MODULE_NAME "builtins" +#endif +#if PY_MAJOR_VERSION >= 3 + #define Py_TPFLAGS_CHECKTYPES 0 + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#if (PY_VERSION_HEX < 0x02060000) || (PY_MAJOR_VERSION >= 3) + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBaseString_Type PyUnicode_Type + #define PyString_Type PyBytes_Type + #define PyString_CheckExact PyBytes_CheckExact + #define PyInt_Type PyLong_Type + #define PyInt_Check(op) PyLong_Check(op) + #define PyInt_CheckExact(op) PyLong_CheckExact(op) + #define PyInt_FromString PyLong_FromString + #define PyInt_FromUnicode PyLong_FromUnicode + #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t + #define PyInt_FromSsize_t PyLong_FromSsize_t + #define PyInt_AsLong PyLong_AsLong + #define PyInt_AS_LONG PyLong_AS_LONG + #define PyInt_AsSsize_t PyLong_AsSsize_t + #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask + #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#else + #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) + #define PyBytes_Type PyString_Type +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyMethod_New(func, self, klass) PyInstanceMethod_New(func) +#endif +#if !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#else + #define _USE_MATH_DEFINES +#endif +#if PY_VERSION_HEX < 0x02050000 + #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),((char *)(n))) + #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),((char *)(n)),(a)) + #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),((char *)(n))) +#else + #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),(n)) + #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),(n),(a)) + #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),(n)) +#endif +#if PY_VERSION_HEX < 0x02050000 + #define __Pyx_NAMESTR(n) ((char *)(n)) + #define __Pyx_DOCSTR(n) ((char *)(n)) +#else + #define __Pyx_NAMESTR(n) (n) + #define __Pyx_DOCSTR(n) (n) +#endif +#ifdef __cplusplus +#define __PYX_EXTERN_C extern "C" +#else +#define __PYX_EXTERN_C extern +#endif +#include +#define __PYX_HAVE_API__msgpack +#include "stdlib.h" +#include "string.h" +#include "pack.h" +#include "unpack.h" +#define __PYX_USE_C99_COMPLEX defined(_Complex_I) + + +#ifdef __GNUC__ +#define INLINE __inline__ +#elif _WIN32 +#define INLINE __inline +#else +#define INLINE +#endif + +typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern; char is_identifier;} __Pyx_StringTabEntry; /*proto*/ + + + +static int __pyx_skip_dispatch = 0; + + +/* Type Conversion Predeclarations */ + +#if PY_MAJOR_VERSION < 3 +#define __Pyx_PyBytes_FromString PyString_FromString +#define __Pyx_PyBytes_FromStringAndSize PyString_FromStringAndSize +#define __Pyx_PyBytes_AsString PyString_AsString +#else +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +#define __Pyx_PyBytes_AsString PyBytes_AsString +#endif + +#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False)) +static INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x); + +#if !defined(T_PYSSIZET) +#if PY_VERSION_HEX < 0x02050000 +#define T_PYSSIZET T_INT +#elif !defined(T_LONGLONG) +#define T_PYSSIZET \ + ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ + ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : -1)) +#else +#define T_PYSSIZET \ + ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ + ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : \ + ((sizeof(Py_ssize_t) == sizeof(PY_LONG_LONG)) ? T_LONGLONG : -1))) +#endif +#endif + +#if !defined(T_SIZET) +#if !defined(T_ULONGLONG) +#define T_SIZET \ + ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ + ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : -1)) +#else +#define T_SIZET \ + ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ + ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : \ + ((sizeof(size_t) == sizeof(unsigned PY_LONG_LONG)) ? T_ULONGLONG : -1))) +#endif +#endif + +static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); +static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*); + +#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) + + +#ifdef __GNUC__ +/* Test for GCC > 2.95 */ +#if __GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)) +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else /* __GNUC__ > 2 ... */ +#define likely(x) (x) +#define unlikely(x) (x) +#endif /* __GNUC__ > 2 ... */ +#else /* __GNUC__ */ +#define likely(x) (x) +#define unlikely(x) (x) +#endif /* __GNUC__ */ + +static PyObject *__pyx_m; +static PyObject *__pyx_b; +static PyObject *__pyx_empty_tuple; +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * __pyx_cfilenm= __FILE__; +static const char *__pyx_filename; +static const char **__pyx_f; + + +#ifdef CYTHON_REFNANNY +typedef struct { + void (*INCREF)(void*, PyObject*, int); + void (*DECREF)(void*, PyObject*, int); + void (*GOTREF)(void*, PyObject*, int); + void (*GIVEREF)(void*, PyObject*, int); + void* (*NewContext)(const char*, int, const char*); + void (*FinishContext)(void**); +} __Pyx_RefnannyAPIStruct; +static __Pyx_RefnannyAPIStruct *__Pyx_Refnanny = NULL; +#define __Pyx_ImportRefcountAPI(name) (__Pyx_RefnannyAPIStruct *) PyCObject_Import((char *)name, (char *)"RefnannyAPI") +#define __Pyx_INCREF(r) __Pyx_Refnanny->INCREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_DECREF(r) __Pyx_Refnanny->DECREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_GOTREF(r) __Pyx_Refnanny->GOTREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_GIVEREF(r) __Pyx_Refnanny->GIVEREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_XDECREF(r) if((r) == NULL) ; else __Pyx_DECREF(r) +#define __Pyx_SetupRefcountContext(name) void* __pyx_refchk = __Pyx_Refnanny->NewContext((name), __LINE__, __FILE__) +#define __Pyx_FinishRefcountContext() __Pyx_Refnanny->FinishContext(&__pyx_refchk) +#else +#define __Pyx_INCREF(r) Py_INCREF(r) +#define __Pyx_DECREF(r) Py_DECREF(r) +#define __Pyx_GOTREF(r) +#define __Pyx_GIVEREF(r) +#define __Pyx_XDECREF(r) Py_XDECREF(r) +#define __Pyx_SetupRefcountContext(name) +#define __Pyx_FinishRefcountContext() +#endif /* CYTHON_REFNANNY */ +#define __Pyx_XGIVEREF(r) if((r) == NULL) ; else __Pyx_GIVEREF(r) +#define __Pyx_XGOTREF(r) if((r) == NULL) ; else __Pyx_GOTREF(r) + +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, PyObject* kw_name); /*proto*/ + +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); /*proto*/ + +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, const char* function_name); /*proto*/ + +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list); /*proto*/ + +static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name); /*proto*/ + +static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); + +static INLINE void __Pyx_RaiseTooManyValuesError(void); + +static PyObject *__Pyx_UnpackItem(PyObject *, Py_ssize_t index); /*proto*/ +static int __Pyx_EndUnpack(PyObject *); /*proto*/ + +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ + +static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ +static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ + +static INLINE int __Pyx_StrEq(const char *, const char *); /*proto*/ + +static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject *); + +static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject *); + +static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject *); + +static INLINE char __Pyx_PyInt_AsChar(PyObject *); + +static INLINE short __Pyx_PyInt_AsShort(PyObject *); + +static INLINE int __Pyx_PyInt_AsInt(PyObject *); + +static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject *); + +static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject *); + +static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject *); + +static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject *); + +static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject *); + +static INLINE long __Pyx_PyInt_AsLong(PyObject *); + +static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject *); + +static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject *); + +static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); + +static void __Pyx_WriteUnraisable(const char *name); /*proto*/ + +static void __Pyx_AddTraceback(const char *funcname); /*proto*/ + +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ + +/* Type declarations */ + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":39 + * cdef int BUFF_SIZE=2*1024 + * + * cdef class Packer: # <<<<<<<<<<<<<< + * """Packer that pack data into strm. + * + */ + +struct __pyx_obj_7msgpack_Packer { + PyObject_HEAD + char *buff; + unsigned int length; + unsigned int allocated; + struct msgpack_packer pk; + PyObject *strm; +}; + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":193 + * return unpacks(packed) + * + * cdef class Unpacker: # <<<<<<<<<<<<<< + * """Do nothing. This function is for symmetric to Packer""" + * unpack = staticmethod(unpacks) + */ + +struct __pyx_obj_7msgpack_Unpacker { + PyObject_HEAD +}; +/* Module declarations from msgpack */ + +static PyTypeObject *__pyx_ptype_7msgpack_Packer = 0; +static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; +static int __pyx_v_7msgpack_BUFF_SIZE; +static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ +#define __Pyx_MODULE_NAME "msgpack" +int __pyx_module_is_main_msgpack = 0; + +/* Implementation of msgpack */ +static char __pyx_k___main__[] = "__main__"; +static PyObject *__pyx_kp___main__; +static char __pyx_k___init__[] = "__init__"; +static PyObject *__pyx_kp___init__; +static char __pyx_k_flush[] = "flush"; +static PyObject *__pyx_kp_flush; +static char __pyx_k_pack_list[] = "pack_list"; +static PyObject *__pyx_kp_pack_list; +static char __pyx_k_pack_dict[] = "pack_dict"; +static PyObject *__pyx_kp_pack_dict; +static char __pyx_k_pack[] = "pack"; +static PyObject *__pyx_kp_pack; +static char __pyx_k_unpack[] = "unpack"; +static PyObject *__pyx_kp_unpack; +static char __pyx_k_strm[] = "strm"; +static PyObject *__pyx_kp_strm; +static char __pyx_k_size[] = "size"; +static PyObject *__pyx_kp_size; +static char __pyx_k_len[] = "len"; +static PyObject *__pyx_kp_len; +static char __pyx_k_o[] = "o"; +static PyObject *__pyx_kp_o; +static char __pyx_k_stream[] = "stream"; +static PyObject *__pyx_kp_stream; +static char __pyx_k_packed_bytes[] = "packed_bytes"; +static PyObject *__pyx_kp_packed_bytes; +static char __pyx_k_cStringIO[] = "cStringIO"; +static PyObject *__pyx_kp_cStringIO; +static char __pyx_k_StringIO[] = "StringIO"; +static PyObject *__pyx_kp_StringIO; +static char __pyx_k_staticmethod[] = "staticmethod"; +static PyObject *__pyx_kp_staticmethod; +static char __pyx_k_unpacks[] = "unpacks"; +static PyObject *__pyx_kp_unpacks; +static char __pyx_k_write[] = "write"; +static PyObject *__pyx_kp_write; +static char __pyx_k_1[] = "flush"; +static PyObject *__pyx_kp_1; +static char __pyx_k_encode[] = "encode"; +static PyObject *__pyx_kp_encode; +static char __pyx_k_iteritems[] = "iteritems"; +static PyObject *__pyx_kp_iteritems; +static char __pyx_k_TypeError[] = "TypeError"; +static PyObject *__pyx_kp_TypeError; +static char __pyx_k_getvalue[] = "getvalue"; +static PyObject *__pyx_kp_getvalue; +static char __pyx_k_read[] = "read"; +static PyObject *__pyx_kp_read; +static PyObject *__pyx_builtin_staticmethod; +static PyObject *__pyx_builtin_TypeError; +static PyObject *__pyx_kp_2; +static PyObject *__pyx_kp_3; +static char __pyx_k_2[] = "utf-8"; +static char __pyx_k_3[] = "can't serialize %r"; + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":51 + * cdef object strm + * + * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< + * if size <= 0: + * size = BUFF_SIZE + */ + +static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_strm = 0; + int __pyx_v_size; + int __pyx_r; + int __pyx_t_1; + static PyObject **__pyx_pyargnames[] = {&__pyx_kp_strm,&__pyx_kp_size,0}; + __Pyx_SetupRefcountContext("__init__"); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); + PyObject* values[2] = {0,0}; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_strm); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (kw_args > 1) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_size); + if (unlikely(value)) { values[1] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "__init__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + __pyx_v_strm = values[0]; + if (values[1]) { + __pyx_v_size = __Pyx_PyInt_AsInt(values[1]); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } else { + __pyx_v_size = 0; + } + } else { + __pyx_v_size = 0; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: __pyx_v_size = __Pyx_PyInt_AsInt(PyTuple_GET_ITEM(__pyx_args, 1)); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + case 1: __pyx_v_strm = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("msgpack.Packer.__init__"); + return -1; + __pyx_L4_argument_unpacking_done:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":52 + * + * def __init__(self, strm, int size=0): + * if size <= 0: # <<<<<<<<<<<<<< + * size = BUFF_SIZE + * + */ + __pyx_t_1 = (__pyx_v_size <= 0); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":53 + * def __init__(self, strm, int size=0): + * if size <= 0: + * size = BUFF_SIZE # <<<<<<<<<<<<<< + * + * self.strm = strm + */ + __pyx_v_size = __pyx_v_7msgpack_BUFF_SIZE; + goto __pyx_L6; + } + __pyx_L6:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":55 + * size = BUFF_SIZE + * + * self.strm = strm # <<<<<<<<<<<<<< + * self.buff = malloc(size) + * self.allocated = size + */ + __Pyx_INCREF(__pyx_v_strm); + __Pyx_GIVEREF(__pyx_v_strm); + __Pyx_GOTREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); + __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":56 + * + * self.strm = strm + * self.buff = malloc(size) # <<<<<<<<<<<<<< + * self.allocated = size + * self.length = 0 + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":57 + * self.strm = strm + * self.buff = malloc(size) + * self.allocated = size # <<<<<<<<<<<<<< + * self.length = 0 + * + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":58 + * self.buff = malloc(size) + * self.allocated = size + * self.length = 0 # <<<<<<<<<<<<<< + * + * msgpack_packer_init(&self.pk, self, _packer_write) + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":60 + * self.length = 0 + * + * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< + * + * + */ + msgpack_packer_init((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), ((void *)__pyx_v_self), ((int (*)(void *, const char*, unsigned int))__pyx_f_7msgpack__packer_write)); + + __pyx_r = 0; + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":63 + * + * + * def flush(self): # <<<<<<<<<<<<<< + * """Flash local buffer and output stream if it has 'flush()' method.""" + * if self.length > 0: + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ +static char __pyx_doc_7msgpack_6Packer_flush[] = "Flash local buffer and output stream if it has 'flush()' method."; +static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused) { + PyObject *__pyx_r = NULL; + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + __Pyx_SetupRefcountContext("flush"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":65 + * def flush(self): + * """Flash local buffer and output stream if it has 'flush()' method.""" + * if self.length > 0: # <<<<<<<<<<<<<< + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + * self.length = 0 + */ + __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":66 + * """Flash local buffer and output stream if it has 'flush()' method.""" + * if self.length > 0: + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< + * self.length = 0 + * if hasattr(self.strm, 'flush'): + */ + __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_4)); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":67 + * if self.length > 0: + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + * self.length = 0 # <<<<<<<<<<<<<< + * if hasattr(self.strm, 'flush'): + * self.strm.flush() + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; + goto __pyx_L5; + } + __pyx_L5:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":68 + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + * self.length = 0 + * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< + * self.strm.flush() + * + */ + __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_1); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":69 + * self.length = 0 + * if hasattr(self.strm, 'flush'): + * self.strm.flush() # <<<<<<<<<<<<<< + * + * def pack_list(self, len): + */ + __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + goto __pyx_L6; + } + __pyx_L6:; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("msgpack.Packer.flush"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":71 + * self.strm.flush() + * + * def pack_list(self, len): # <<<<<<<<<<<<<< + * """Start packing sequential objects. + * + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ +static char __pyx_doc_7msgpack_6Packer_pack_list[] = "Start packing sequential objects.\n\n Example:\n\n packer.pack_list(2)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack(['foo', 'bar'])\n "; +static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { + PyObject *__pyx_r = NULL; + size_t __pyx_t_1; + __Pyx_SetupRefcountContext("pack_list"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":84 + * packer.pack(['foo', 'bar']) + * """ + * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< + * + * def pack_dict(self, len): + */ + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("msgpack.Packer.pack_list"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":86 + * msgpack_pack_array(&self.pk, len) + * + * def pack_dict(self, len): # <<<<<<<<<<<<<< + * """Start packing key-value objects. + * + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ +static char __pyx_doc_7msgpack_6Packer_pack_dict[] = "Start packing key-value objects.\n\n Example:\n\n packer.pack_dict(1)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack({'foo', 'bar'})\n "; +static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { + PyObject *__pyx_r = NULL; + size_t __pyx_t_1; + __Pyx_SetupRefcountContext("pack_dict"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":99 + * packer.pack({'foo', 'bar'}) + * """ + * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< + * + * def pack(self, object o): + */ + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("msgpack.Packer.pack_dict"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":101 + * msgpack_pack_map(&self.pk, len) + * + * def pack(self, object o): # <<<<<<<<<<<<<< + * cdef long long intval + * cdef double fval + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o); /*proto*/ +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o) { + PY_LONG_LONG __pyx_v_intval; + double __pyx_v_fval; + char *__pyx_v_rawval; + PyObject *__pyx_v_k; + PyObject *__pyx_v_v; + PyObject *__pyx_r = NULL; + PyObject *__pyx_1 = 0; + PyObject *__pyx_2 = 0; + PyObject *__pyx_3 = 0; + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PY_LONG_LONG __pyx_t_3; + char *__pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + int __pyx_t_9; + int __pyx_t_10; + __Pyx_SetupRefcountContext("pack"); + __Pyx_INCREF(__pyx_v_o); + __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); + __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":106 + * cdef char* rawval + * + * if o is None: # <<<<<<<<<<<<<< + * msgpack_pack_nil(&self.pk) + * elif o is True: + */ + __pyx_t_1 = (__pyx_v_o == Py_None); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":107 + * + * if o is None: + * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< + * elif o is True: + * msgpack_pack_true(&self.pk) + */ + msgpack_pack_nil((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":108 + * if o is None: + * msgpack_pack_nil(&self.pk) + * elif o is True: # <<<<<<<<<<<<<< + * msgpack_pack_true(&self.pk) + * elif o is False: + */ + __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = (__pyx_v_o == __pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":109 + * msgpack_pack_nil(&self.pk) + * elif o is True: + * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< + * elif o is False: + * msgpack_pack_false(&self.pk) + */ + msgpack_pack_true((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":110 + * elif o is True: + * msgpack_pack_true(&self.pk) + * elif o is False: # <<<<<<<<<<<<<< + * msgpack_pack_false(&self.pk) + * elif isinstance(o, long): + */ + __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = (__pyx_v_o == __pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":111 + * msgpack_pack_true(&self.pk) + * elif o is False: + * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< + * elif isinstance(o, long): + * intval = o + */ + msgpack_pack_false((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":112 + * elif o is False: + * msgpack_pack_false(&self.pk) + * elif isinstance(o, long): # <<<<<<<<<<<<<< + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":113 + * msgpack_pack_false(&self.pk) + * elif isinstance(o, long): + * intval = o # <<<<<<<<<<<<<< + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, int): + */ + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_intval = __pyx_t_3; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":114 + * elif isinstance(o, long): + * intval = o + * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< + * elif isinstance(o, int): + * intval = o + */ + msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":115 + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, int): # <<<<<<<<<<<<<< + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":116 + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, int): + * intval = o # <<<<<<<<<<<<<< + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, float): + */ + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_intval = __pyx_t_3; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":117 + * elif isinstance(o, int): + * intval = o + * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< + * elif isinstance(o, float): + * fval = 9 + */ + msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":118 + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, float): # <<<<<<<<<<<<<< + * fval = 9 + * msgpack_pack_double(&self.pk, fval) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":119 + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, float): + * fval = 9 # <<<<<<<<<<<<<< + * msgpack_pack_double(&self.pk, fval) + * elif isinstance(o, str): + */ + __pyx_v_fval = 9; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":120 + * elif isinstance(o, float): + * fval = 9 + * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< + * elif isinstance(o, str): + * rawval = o + */ + msgpack_pack_double((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_fval); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":121 + * fval = 9 + * msgpack_pack_double(&self.pk, fval) + * elif isinstance(o, str): # <<<<<<<<<<<<<< + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":122 + * msgpack_pack_double(&self.pk, fval) + * elif isinstance(o, str): + * rawval = o # <<<<<<<<<<<<<< + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + */ + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_4; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":123 + * elif isinstance(o, str): + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, unicode): + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":124 + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< + * elif isinstance(o, unicode): + * o = o.encode('utf-8') + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":125 + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, unicode): # <<<<<<<<<<<<<< + * o = o.encode('utf-8') + * rawval = o + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":126 + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, unicode): + * o = o.encode('utf-8') # <<<<<<<<<<<<<< + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_6)); + __Pyx_INCREF(__pyx_kp_2); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_2); + __Pyx_GIVEREF(__pyx_kp_2); + __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_v_o); + __pyx_v_o = __pyx_t_7; + __pyx_t_7 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":127 + * elif isinstance(o, unicode): + * o = o.encode('utf-8') + * rawval = o # <<<<<<<<<<<<<< + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + */ + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_4; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":128 + * o = o.encode('utf-8') + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, dict): + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":129 + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< + * elif isinstance(o, dict): + * msgpack_pack_map(&self.pk, len(o)) + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":130 + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, dict): # <<<<<<<<<<<<<< + * msgpack_pack_map(&self.pk, len(o)) + * for k,v in o.iteritems(): + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":131 + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, dict): + * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< + * for k,v in o.iteritems(): + * self.pack(k) + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":132 + * elif isinstance(o, dict): + * msgpack_pack_map(&self.pk, len(o)) + * for k,v in o.iteritems(): # <<<<<<<<<<<<<< + * self.pack(k) + * self.pack(v) + */ + __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { + __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); + } else { + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + for (;;) { + if (likely(PyList_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else { + __pyx_t_6 = PyIter_Next(__pyx_t_7); + if (!__pyx_t_6) { + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + break; + } + __Pyx_GOTREF(__pyx_t_6); + } + if (PyTuple_CheckExact(__pyx_t_6) && likely(PyTuple_GET_SIZE(__pyx_t_6) == 2)) { + PyObject* tuple = __pyx_t_6; + __pyx_2 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_2); + __pyx_3 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_3); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_v_k); + __pyx_v_k = __pyx_2; + __pyx_2 = 0; + __Pyx_DECREF(__pyx_v_v); + __pyx_v_v = __pyx_3; + __pyx_3 = 0; + } else { + __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_2); + __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_3); + if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + __Pyx_DECREF(__pyx_v_k); + __pyx_v_k = __pyx_2; + __pyx_2 = 0; + __Pyx_DECREF(__pyx_v_v); + __pyx_v_v = __pyx_3; + __pyx_3 = 0; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":133 + * msgpack_pack_map(&self.pk, len(o)) + * for k,v in o.iteritems(): + * self.pack(k) # <<<<<<<<<<<<<< + * self.pack(v) + * elif isinstance(o, tuple) or isinstance(o, list): + */ + __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_k); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); + __Pyx_GIVEREF(__pyx_v_k); + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":134 + * for k,v in o.iteritems(): + * self.pack(k) + * self.pack(v) # <<<<<<<<<<<<<< + * elif isinstance(o, tuple) or isinstance(o, list): + * msgpack_pack_array(&self.pk, len(o)) + */ + __pyx_t_8 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_v); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); + __Pyx_GIVEREF(__pyx_v_v); + __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":135 + * self.pack(k) + * self.pack(v) + * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< + * msgpack_pack_array(&self.pk, len(o)) + * for v in o: + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyTuple_Type))); + if (!__pyx_t_1) { + __pyx_t_9 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); + __pyx_t_10 = __pyx_t_9; + } else { + __pyx_t_10 = __pyx_t_1; + } + if (__pyx_t_10) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":136 + * self.pack(v) + * elif isinstance(o, tuple) or isinstance(o, list): + * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< + * for v in o: + * self.pack(v) + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":137 + * elif isinstance(o, tuple) or isinstance(o, list): + * msgpack_pack_array(&self.pk, len(o)) + * for v in o: # <<<<<<<<<<<<<< + * self.pack(v) + * else: + */ + if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { + __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); + } else { + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + } + for (;;) { + if (likely(PyList_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else { + __pyx_t_6 = PyIter_Next(__pyx_t_7); + if (!__pyx_t_6) { + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + break; + } + __Pyx_GOTREF(__pyx_t_6); + } + __Pyx_DECREF(__pyx_v_v); + __pyx_v_v = __pyx_t_6; + __pyx_t_6 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":138 + * msgpack_pack_array(&self.pk, len(o)) + * for v in o: + * self.pack(v) # <<<<<<<<<<<<<< + * else: + * # TODO: Serialize with defalt() like simplejson. + */ + __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_v); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); + __Pyx_GIVEREF(__pyx_v_v); + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L5; + } + /*else*/ { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":141 + * else: + * # TODO: Serialize with defalt() like simplejson. + * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + */ + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_7)); + __Pyx_INCREF(__pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); + __Pyx_GIVEREF(__pyx_v_o); + __pyx_t_8 = PyNumber_Remainder(__pyx_kp_3, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; + __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } + __pyx_L5:; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_2); + __Pyx_XDECREF(__pyx_3); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("msgpack.Packer.pack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_k); + __Pyx_DECREF(__pyx_v_v); + __Pyx_DECREF(__pyx_v_o); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":143 + * raise TypeError, "can't serialize %r" % (o,) + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< + * if packer.length + l > packer.allocated: + * if packer.length > 0: + */ + +static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__pyx_v_packer, const char* __pyx_v_b, unsigned int __pyx_v_l) { + int __pyx_r; + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + __Pyx_SetupRefcountContext("_packer_write"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":144 + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< + * if packer.length > 0: + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + */ + __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":145 + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + * if packer.length + l > packer.allocated: + * if packer.length > 0: # <<<<<<<<<<<<<< + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + * if l > 64: + */ + __pyx_t_1 = (__pyx_v_packer->length > 0); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":146 + * if packer.length + l > packer.allocated: + * if packer.length > 0: + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< + * if l > 64: + * packer.strm.write(PyString_FromStringAndSize(b, l)) + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_4)); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L4; + } + __pyx_L4:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":147 + * if packer.length > 0: + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + * if l > 64: # <<<<<<<<<<<<<< + * packer.strm.write(PyString_FromStringAndSize(b, l)) + * packer.length = 0 + */ + __pyx_t_1 = (__pyx_v_l > 64); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":148 + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + * if l > 64: + * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< + * packer.length = 0 + * else: + */ + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":149 + * if l > 64: + * packer.strm.write(PyString_FromStringAndSize(b, l)) + * packer.length = 0 # <<<<<<<<<<<<<< + * else: + * memcpy(packer.buff, b, l) + */ + __pyx_v_packer->length = 0; + goto __pyx_L5; + } + /*else*/ { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":151 + * packer.length = 0 + * else: + * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< + * packer.length = l + * else: + */ + memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":152 + * else: + * memcpy(packer.buff, b, l) + * packer.length = l # <<<<<<<<<<<<<< + * else: + * memcpy(packer.buff + packer.length, b, l) + */ + __pyx_v_packer->length = __pyx_v_l; + } + __pyx_L5:; + goto __pyx_L3; + } + /*else*/ { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":154 + * packer.length = l + * else: + * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< + * packer.length += l + * return 0 + */ + memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":155 + * else: + * memcpy(packer.buff + packer.length, b, l) + * packer.length += l # <<<<<<<<<<<<<< + * return 0 + * + */ + __pyx_v_packer->length += __pyx_v_l; + } + __pyx_L3:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":156 + * memcpy(packer.buff + packer.length, b, l) + * packer.length += l + * return 0 # <<<<<<<<<<<<<< + * + * def pack(object o, object stream): + */ + __pyx_r = 0; + goto __pyx_L0; + + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_WriteUnraisable("msgpack._packer_write"); + __pyx_r = 0; + __pyx_L0:; + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":158 + * return 0 + * + * def pack(object o, object stream): # <<<<<<<<<<<<<< + * packer = Packer(stream) + * packer.pack(o) + */ + +static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_o = 0; + PyObject *__pyx_v_stream = 0; + PyObject *__pyx_v_packer; + PyObject *__pyx_r = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + static PyObject **__pyx_pyargnames[] = {&__pyx_kp_o,&__pyx_kp_stream,0}; + __Pyx_SetupRefcountContext("pack"); + __pyx_self = __pyx_self; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); + PyObject* values[2] = {0,0}; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_o); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + values[1] = PyDict_GetItem(__pyx_kwds, __pyx_kp_stream); + if (likely(values[1])) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + __pyx_v_o = values[0]; + __pyx_v_stream = values[1]; + } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { + goto __pyx_L5_argtuple_error; + } else { + __pyx_v_o = PyTuple_GET_ITEM(__pyx_args, 0); + __pyx_v_stream = PyTuple_GET_ITEM(__pyx_args, 1); + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("msgpack.pack"); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":159 + * + * def pack(object o, object stream): + * packer = Packer(stream) # <<<<<<<<<<<<<< + * packer.pack(o) + * packer.flush() + */ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_stream); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_stream); + __Pyx_GIVEREF(__pyx_v_stream); + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_v_packer); + __pyx_v_packer = __pyx_t_2; + __pyx_t_2 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":160 + * def pack(object o, object stream): + * packer = Packer(stream) + * packer.pack(o) # <<<<<<<<<<<<<< + * packer.flush() + * + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); + __Pyx_GIVEREF(__pyx_v_o); + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":161 + * packer = Packer(stream) + * packer.pack(o) + * packer.flush() # <<<<<<<<<<<<<< + * + * def packs(object o): + */ + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("msgpack.pack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_packer); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":163 + * packer.flush() + * + * def packs(object o): # <<<<<<<<<<<<<< + * buf = StringIO() + * packer = Packer(buf) + */ + +static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o); /*proto*/ +static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o) { + PyObject *__pyx_v_buf; + PyObject *__pyx_v_packer; + PyObject *__pyx_r = NULL; + PyObject *__pyx_1 = 0; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + __Pyx_SetupRefcountContext("packs"); + __pyx_self = __pyx_self; + __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); + __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":164 + * + * def packs(object o): + * buf = StringIO() # <<<<<<<<<<<<<< + * packer = Packer(buf) + * packer.pack(o) + */ + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + __Pyx_DECREF(__pyx_v_buf); + __pyx_v_buf = __pyx_t_1; + __pyx_t_1 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":165 + * def packs(object o): + * buf = StringIO() + * packer = Packer(buf) # <<<<<<<<<<<<<< + * packer.pack(o) + * packer.flush() + */ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_buf); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_buf); + __Pyx_GIVEREF(__pyx_v_buf); + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_v_packer); + __pyx_v_packer = __pyx_t_2; + __pyx_t_2 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":166 + * buf = StringIO() + * packer = Packer(buf) + * packer.pack(o) # <<<<<<<<<<<<<< + * packer.flush() + * return buf.getvalue() + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); + __Pyx_GIVEREF(__pyx_v_o); + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":167 + * packer = Packer(buf) + * packer.pack(o) + * packer.flush() # <<<<<<<<<<<<<< + * return buf.getvalue() + * + */ + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":168 + * packer.pack(o) + * packer.flush() + * return buf.getvalue() # <<<<<<<<<<<<<< + * + * cdef extern from "unpack.h": + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("msgpack.packs"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_buf); + __Pyx_DECREF(__pyx_v_packer); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":179 + * + * + * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< + * """Unpack packed_bytes to object. Returns unpacked object.""" + * cdef const_char_ptr p = packed_bytes + */ + +static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes); /*proto*/ +static char __pyx_doc_7msgpack_unpacks[] = "Unpack packed_bytes to object. Returns unpacked object."; +static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes) { + const char* __pyx_v_p; + template_context __pyx_v_ctx; + size_t __pyx_v_off; + PyObject *__pyx_r = NULL; + const char* __pyx_t_1; + Py_ssize_t __pyx_t_2; + PyObject *__pyx_t_3; + __Pyx_SetupRefcountContext("unpacks"); + __pyx_self = __pyx_self; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":181 + * def unpacks(object packed_bytes): + * """Unpack packed_bytes to object. Returns unpacked object.""" + * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< + * cdef template_context ctx + * cdef size_t off = 0 + */ + __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 181; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_p = __pyx_t_1; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":183 + * cdef const_char_ptr p = packed_bytes + * cdef template_context ctx + * cdef size_t off = 0 # <<<<<<<<<<<<<< + * template_init(&ctx) + * template_execute(&ctx, p, len(packed_bytes), &off) + */ + __pyx_v_off = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":184 + * cdef template_context ctx + * cdef size_t off = 0 + * template_init(&ctx) # <<<<<<<<<<<<<< + * template_execute(&ctx, p, len(packed_bytes), &off) + * return template_data(&ctx) + */ + template_init((&__pyx_v_ctx)); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":185 + * cdef size_t off = 0 + * template_init(&ctx) + * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< + * return template_data(&ctx) + * + */ + __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 185; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":186 + * template_init(&ctx) + * template_execute(&ctx, p, len(packed_bytes), &off) + * return template_data(&ctx) # <<<<<<<<<<<<<< + * + * def unpack(object stream): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_3 = template_data((&__pyx_v_ctx)); + __Pyx_INCREF(((PyObject *)__pyx_t_3)); + __pyx_r = ((PyObject *)__pyx_t_3); + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("msgpack.unpacks"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":188 + * return template_data(&ctx) + * + * def unpack(object stream): # <<<<<<<<<<<<<< + * """unpack from stream.""" + * packed = stream.read() + */ + +static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream); /*proto*/ +static char __pyx_doc_7msgpack_unpack[] = "unpack from stream."; +static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream) { + PyObject *__pyx_v_packed; + PyObject *__pyx_r = NULL; + PyObject *__pyx_1 = 0; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + __Pyx_SetupRefcountContext("unpack"); + __pyx_self = __pyx_self; + __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":190 + * def unpack(object stream): + * """unpack from stream.""" + * packed = stream.read() # <<<<<<<<<<<<<< + * return unpacks(packed) + * + */ + __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_v_packed); + __pyx_v_packed = __pyx_t_2; + __pyx_t_2 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":191 + * """unpack from stream.""" + * packed = stream.read() + * return unpacks(packed) # <<<<<<<<<<<<<< + * + * cdef class Unpacker: + */ + __Pyx_XDECREF(__pyx_r); + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_packed); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_packed); + __Pyx_GIVEREF(__pyx_v_packed); + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("msgpack.unpack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_packed); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +static PyObject *__pyx_tp_new_7msgpack_Packer(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_7msgpack_Packer *p; + PyObject *o = (*t->tp_alloc)(t, 0); + if (!o) return 0; + p = ((struct __pyx_obj_7msgpack_Packer *)o); + p->strm = Py_None; Py_INCREF(Py_None); + return o; +} + +static void __pyx_tp_dealloc_7msgpack_Packer(PyObject *o) { + struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; + Py_XDECREF(p->strm); + (*Py_TYPE(o)->tp_free)(o); +} + +static int __pyx_tp_traverse_7msgpack_Packer(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; + if (p->strm) { + e = (*v)(p->strm, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_7msgpack_Packer(PyObject *o) { + struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; + PyObject* tmp; + tmp = ((PyObject*)p->strm); + p->strm = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static struct PyMethodDef __pyx_methods_7msgpack_Packer[] = { + {__Pyx_NAMESTR("flush"), (PyCFunction)__pyx_pf_7msgpack_6Packer_flush, METH_NOARGS, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_flush)}, + {__Pyx_NAMESTR("pack_list"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_list, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_list)}, + {__Pyx_NAMESTR("pack_dict"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_dict, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_dict)}, + {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_O, __Pyx_DOCSTR(0)}, + {0, 0, 0, 0} +}; + +static PyNumberMethods __pyx_tp_as_number_Packer = { + 0, /*nb_add*/ + 0, /*nb_subtract*/ + 0, /*nb_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_divide*/ + #endif + 0, /*nb_remainder*/ + 0, /*nb_divmod*/ + 0, /*nb_power*/ + 0, /*nb_negative*/ + 0, /*nb_positive*/ + 0, /*nb_absolute*/ + 0, /*nb_nonzero*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + 0, /*nb_and*/ + 0, /*nb_xor*/ + 0, /*nb_or*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_coerce*/ + #endif + 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else + 0, /*nb_long*/ + #endif + 0, /*nb_float*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_oct*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*nb_hex*/ + #endif + 0, /*nb_inplace_add*/ + 0, /*nb_inplace_subtract*/ + 0, /*nb_inplace_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_inplace_divide*/ + #endif + 0, /*nb_inplace_remainder*/ + 0, /*nb_inplace_power*/ + 0, /*nb_inplace_lshift*/ + 0, /*nb_inplace_rshift*/ + 0, /*nb_inplace_and*/ + 0, /*nb_inplace_xor*/ + 0, /*nb_inplace_or*/ + 0, /*nb_floor_divide*/ + 0, /*nb_true_divide*/ + 0, /*nb_inplace_floor_divide*/ + 0, /*nb_inplace_true_divide*/ + #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) + 0, /*nb_index*/ + #endif +}; + +static PySequenceMethods __pyx_tp_as_sequence_Packer = { + 0, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_Packer = { + 0, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_Packer = { + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getreadbuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getwritebuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getsegcount*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getcharbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_getbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_releasebuffer*/ + #endif +}; + +PyTypeObject __pyx_type_7msgpack_Packer = { + PyVarObject_HEAD_INIT(0, 0) + __Pyx_NAMESTR("msgpack.Packer"), /*tp_name*/ + sizeof(struct __pyx_obj_7msgpack_Packer), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_7msgpack_Packer, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + &__pyx_tp_as_number_Packer, /*tp_as_number*/ + &__pyx_tp_as_sequence_Packer, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_Packer, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_Packer, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __Pyx_DOCSTR("Packer that pack data into strm.\n\n strm must have `write(bytes)` method.\n size specifies local buffer size.\n "), /*tp_doc*/ + __pyx_tp_traverse_7msgpack_Packer, /*tp_traverse*/ + __pyx_tp_clear_7msgpack_Packer, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_7msgpack_Packer, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + __pyx_pf_7msgpack_6Packer___init__, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_7msgpack_Packer, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ +}; + +static PyObject *__pyx_tp_new_7msgpack_Unpacker(PyTypeObject *t, PyObject *a, PyObject *k) { + PyObject *o = (*t->tp_alloc)(t, 0); + if (!o) return 0; + return o; +} + +static void __pyx_tp_dealloc_7msgpack_Unpacker(PyObject *o) { + (*Py_TYPE(o)->tp_free)(o); +} + +static struct PyMethodDef __pyx_methods_7msgpack_Unpacker[] = { + {0, 0, 0, 0} +}; + +static PyNumberMethods __pyx_tp_as_number_Unpacker = { + 0, /*nb_add*/ + 0, /*nb_subtract*/ + 0, /*nb_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_divide*/ + #endif + 0, /*nb_remainder*/ + 0, /*nb_divmod*/ + 0, /*nb_power*/ + 0, /*nb_negative*/ + 0, /*nb_positive*/ + 0, /*nb_absolute*/ + 0, /*nb_nonzero*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + 0, /*nb_and*/ + 0, /*nb_xor*/ + 0, /*nb_or*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_coerce*/ + #endif + 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else + 0, /*nb_long*/ + #endif + 0, /*nb_float*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_oct*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*nb_hex*/ + #endif + 0, /*nb_inplace_add*/ + 0, /*nb_inplace_subtract*/ + 0, /*nb_inplace_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_inplace_divide*/ + #endif + 0, /*nb_inplace_remainder*/ + 0, /*nb_inplace_power*/ + 0, /*nb_inplace_lshift*/ + 0, /*nb_inplace_rshift*/ + 0, /*nb_inplace_and*/ + 0, /*nb_inplace_xor*/ + 0, /*nb_inplace_or*/ + 0, /*nb_floor_divide*/ + 0, /*nb_true_divide*/ + 0, /*nb_inplace_floor_divide*/ + 0, /*nb_inplace_true_divide*/ + #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) + 0, /*nb_index*/ + #endif +}; + +static PySequenceMethods __pyx_tp_as_sequence_Unpacker = { + 0, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_Unpacker = { + 0, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_Unpacker = { + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getreadbuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getwritebuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getsegcount*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getcharbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_getbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_releasebuffer*/ + #endif +}; + +PyTypeObject __pyx_type_7msgpack_Unpacker = { + PyVarObject_HEAD_INIT(0, 0) + __Pyx_NAMESTR("msgpack.Unpacker"), /*tp_name*/ + sizeof(struct __pyx_obj_7msgpack_Unpacker), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_7msgpack_Unpacker, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + &__pyx_tp_as_number_Unpacker, /*tp_as_number*/ + &__pyx_tp_as_sequence_Unpacker, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_Unpacker, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_Unpacker, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ + __Pyx_DOCSTR("Do nothing. This function is for symmetric to Packer"), /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_7msgpack_Unpacker, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_7msgpack_Unpacker, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ +}; + +static struct PyMethodDef __pyx_methods[] = { + {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("packs"), (PyCFunction)__pyx_pf_7msgpack_packs, METH_O, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("unpacks"), (PyCFunction)__pyx_pf_7msgpack_unpacks, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpacks)}, + {__Pyx_NAMESTR("unpack"), (PyCFunction)__pyx_pf_7msgpack_unpack, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpack)}, + {0, 0, 0, 0} +}; + +static void __pyx_init_filenames(void); /*proto*/ + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef __pyx_moduledef = { + PyModuleDef_HEAD_INIT, + __Pyx_NAMESTR("msgpack"), + 0, /* m_doc */ + -1, /* m_size */ + __pyx_methods /* m_methods */, + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ +}; +#endif + +static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, + {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, + {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, + {&__pyx_kp_pack_list, __pyx_k_pack_list, sizeof(__pyx_k_pack_list), 1, 1, 1}, + {&__pyx_kp_pack_dict, __pyx_k_pack_dict, sizeof(__pyx_k_pack_dict), 1, 1, 1}, + {&__pyx_kp_pack, __pyx_k_pack, sizeof(__pyx_k_pack), 1, 1, 1}, + {&__pyx_kp_unpack, __pyx_k_unpack, sizeof(__pyx_k_unpack), 1, 1, 1}, + {&__pyx_kp_strm, __pyx_k_strm, sizeof(__pyx_k_strm), 1, 1, 1}, + {&__pyx_kp_size, __pyx_k_size, sizeof(__pyx_k_size), 1, 1, 1}, + {&__pyx_kp_len, __pyx_k_len, sizeof(__pyx_k_len), 1, 1, 1}, + {&__pyx_kp_o, __pyx_k_o, sizeof(__pyx_k_o), 1, 1, 1}, + {&__pyx_kp_stream, __pyx_k_stream, sizeof(__pyx_k_stream), 1, 1, 1}, + {&__pyx_kp_packed_bytes, __pyx_k_packed_bytes, sizeof(__pyx_k_packed_bytes), 1, 1, 1}, + {&__pyx_kp_cStringIO, __pyx_k_cStringIO, sizeof(__pyx_k_cStringIO), 1, 1, 1}, + {&__pyx_kp_StringIO, __pyx_k_StringIO, sizeof(__pyx_k_StringIO), 1, 1, 1}, + {&__pyx_kp_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 1, 1, 1}, + {&__pyx_kp_unpacks, __pyx_k_unpacks, sizeof(__pyx_k_unpacks), 1, 1, 1}, + {&__pyx_kp_write, __pyx_k_write, sizeof(__pyx_k_write), 1, 1, 1}, + {&__pyx_kp_1, __pyx_k_1, sizeof(__pyx_k_1), 0, 1, 0}, + {&__pyx_kp_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 1, 1, 1}, + {&__pyx_kp_iteritems, __pyx_k_iteritems, sizeof(__pyx_k_iteritems), 1, 1, 1}, + {&__pyx_kp_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 1, 1, 1}, + {&__pyx_kp_getvalue, __pyx_k_getvalue, sizeof(__pyx_k_getvalue), 1, 1, 1}, + {&__pyx_kp_read, __pyx_k_read, sizeof(__pyx_k_read), 1, 1, 1}, + {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 0, 0}, + {&__pyx_kp_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 0}, + {0, 0, 0, 0, 0, 0} +}; +static int __Pyx_InitCachedBuiltins(void) { + __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + return 0; + __pyx_L1_error:; + return -1; +} + +static int __Pyx_InitGlobals(void) { + if (__Pyx_InitStrings(__pyx_string_tab) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + return 0; + __pyx_L1_error:; + return -1; +} + +#if PY_MAJOR_VERSION < 3 +PyMODINIT_FUNC initmsgpack(void); /*proto*/ +PyMODINIT_FUNC initmsgpack(void) +#else +PyMODINIT_FUNC PyInit_msgpack(void); /*proto*/ +PyMODINIT_FUNC PyInit_msgpack(void) +#endif +{ + PyObject *__pyx_1 = 0; + PyObject *__pyx_2 = 0; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + #ifdef CYTHON_REFNANNY + void* __pyx_refchk = NULL; + __Pyx_Refnanny = __Pyx_ImportRefcountAPI("refnanny"); + if (!__Pyx_Refnanny) { + PyErr_Clear(); + __Pyx_Refnanny = __Pyx_ImportRefcountAPI("Cython.Runtime.refnanny"); + if (!__Pyx_Refnanny) + Py_FatalError("failed to import refnanny module"); + } + __pyx_refchk = __Pyx_Refnanny->NewContext("PyMODINIT_FUNC PyInit_msgpack(void)", __LINE__, __FILE__); + #endif + __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + /*--- Library function declarations ---*/ + __pyx_init_filenames(); + /*--- Threads initialization code ---*/ + #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS + #ifdef WITH_THREAD /* Python build with threading support? */ + PyEval_InitThreads(); + #endif + #endif + /*--- Initialize various global constants etc. ---*/ + if (unlikely(__Pyx_InitGlobals() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + /*--- Module creation code ---*/ + #if PY_MAJOR_VERSION < 3 + __pyx_m = Py_InitModule4(__Pyx_NAMESTR("msgpack"), __pyx_methods, 0, 0, PYTHON_API_VERSION); + #else + __pyx_m = PyModule_Create(&__pyx_moduledef); + #endif + if (!__pyx_m) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + #if PY_MAJOR_VERSION < 3 + Py_INCREF(__pyx_m); + #endif + __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); + if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + if (__pyx_module_is_main_msgpack) { + if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + } + /*--- Builtin init code ---*/ + if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_skip_dispatch = 0; + /*--- Global init code ---*/ + /*--- Function export code ---*/ + /*--- Type init code ---*/ + if (PyType_Ready(&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "Packer", (PyObject *)&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_7msgpack_Packer = &__pyx_type_7msgpack_Packer; + if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_7msgpack_Unpacker = &__pyx_type_7msgpack_Unpacker; + /*--- Type import code ---*/ + /*--- Function import code ---*/ + /*--- Execution code ---*/ + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":3 + * # coding: utf-8 + * + * from cStringIO import StringIO # <<<<<<<<<<<<<< + * + * cdef extern from "Python.h": + */ + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_kp_StringIO); + PyList_SET_ITEM(__pyx_t_1, 0, __pyx_kp_StringIO); + __Pyx_GIVEREF(__pyx_kp_StringIO); + __pyx_1 = __Pyx_Import(__pyx_kp_cStringIO, ((PyObject *)__pyx_t_1)); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __pyx_2 = PyObject_GetAttr(__pyx_1, __pyx_kp_StringIO); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_2); + if (PyObject_SetAttr(__pyx_m, __pyx_kp_StringIO, __pyx_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_2); __pyx_2 = 0; + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":37 + * + * + * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< + * + * cdef class Packer: + */ + __pyx_v_7msgpack_BUFF_SIZE = 2048; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":195 + * cdef class Unpacker: + * """Do nothing. This function is for symmetric to Packer""" + * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< + */ + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_1); + __Pyx_GIVEREF(__pyx_1); + __pyx_1 = 0; + __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + PyType_Modified(__pyx_ptype_7msgpack_Unpacker); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_2); + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("msgpack"); + Py_DECREF(__pyx_m); __pyx_m = 0; + __pyx_L0:; + __Pyx_FinishRefcountContext(); + #if PY_MAJOR_VERSION < 3 + return; + #else + return __pyx_m; + #endif +} + +static const char *__pyx_filenames[] = { + "msgpack.pyx", +}; + +/* Runtime support code */ + +static void __pyx_init_filenames(void) { + __pyx_f = __pyx_filenames; +} + +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION >= 3 + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); + #else + "%s() got multiple values for keyword argument '%s'", func_name, + PyString_AS_STRING(kw_name)); + #endif +} + +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *number, *more_or_less; + + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + number = (num_expected == 1) ? "" : "s"; + PyErr_Format(PyExc_TypeError, + #if PY_VERSION_HEX < 0x02050000 + "%s() takes %s %d positional argument%s (%d given)", + #else + "%s() takes %s %zd positional argument%s (%zd given)", + #endif + func_name, more_or_less, num_expected, number, num_found); +} + +static int __Pyx_ParseOptionalKeywords( + PyObject *kwds, + PyObject **argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject *key = 0, *value = 0; + Py_ssize_t pos = 0; + PyObject*** name; + PyObject*** first_kw_arg = argnames + num_pos_args; + + while (PyDict_Next(kwds, &pos, &key, &value)) { + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + values[name-argnames] = value; + } else { + #if PY_MAJOR_VERSION < 3 + if (unlikely(!PyString_CheckExact(key)) && unlikely(!PyString_Check(key))) { + #else + if (unlikely(!PyUnicode_CheckExact(key)) && unlikely(!PyUnicode_Check(key))) { + #endif + goto invalid_keyword_type; + } else { + for (name = first_kw_arg; *name; name++) { + #if PY_MAJOR_VERSION >= 3 + if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && + PyUnicode_Compare(**name, key) == 0) break; + #else + if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && + _PyString_Eq(**name, key)) break; + #endif + } + if (*name) { + values[name-argnames] = value; + } else { + /* unexpected keyword found */ + for (name=argnames; name != first_kw_arg; name++) { + if (**name == key) goto arg_passed_twice; + #if PY_MAJOR_VERSION >= 3 + if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && + PyUnicode_Compare(**name, key) == 0) goto arg_passed_twice; + #else + if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && + _PyString_Eq(**name, key)) goto arg_passed_twice; + #endif + } + if (kwds2) { + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else { + goto invalid_keyword; + } + } + } + } + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, **name); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%s() keywords must be strings", function_name); + goto bad; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%s() got an unexpected keyword argument '%s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif +bad: + return -1; +} + +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list) { + PyObject *__import__ = 0; + PyObject *empty_list = 0; + PyObject *module = 0; + PyObject *global_dict = 0; + PyObject *empty_dict = 0; + PyObject *list; + __import__ = __Pyx_GetAttrString(__pyx_b, "__import__"); + if (!__import__) + goto bad; + if (from_list) + list = from_list; + else { + empty_list = PyList_New(0); + if (!empty_list) + goto bad; + list = empty_list; + } + global_dict = PyModule_GetDict(__pyx_m); + if (!global_dict) + goto bad; + empty_dict = PyDict_New(); + if (!empty_dict) + goto bad; + module = PyObject_CallFunctionObjArgs(__import__, + name, global_dict, empty_dict, list, NULL); +bad: + Py_XDECREF(empty_list); + Py_XDECREF(__import__); + Py_XDECREF(empty_dict); + return module; +} + +static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name) { + PyObject *result; + result = PyObject_GetAttr(dict, name); + if (!result) + PyErr_SetObject(PyExc_NameError, name); + return result; +} + +static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { + PyErr_Format(PyExc_ValueError, + #if PY_VERSION_HEX < 0x02050000 + "need more than %d value%s to unpack", (int)index, + #else + "need more than %zd value%s to unpack", index, + #endif + (index == 1) ? "" : "s"); +} + +static INLINE void __Pyx_RaiseTooManyValuesError(void) { + PyErr_SetString(PyExc_ValueError, "too many values to unpack"); +} + +static PyObject *__Pyx_UnpackItem(PyObject *iter, Py_ssize_t index) { + PyObject *item; + if (!(item = PyIter_Next(iter))) { + if (!PyErr_Occurred()) { + __Pyx_RaiseNeedMoreValuesError(index); + } + } + return item; +} + +static int __Pyx_EndUnpack(PyObject *iter) { + PyObject *item; + if ((item = PyIter_Next(iter))) { + Py_DECREF(item); + __Pyx_RaiseTooManyValuesError(); + return -1; + } + else if (!PyErr_Occurred()) + return 0; + else + return -1; +} + +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb) { + Py_XINCREF(type); + Py_XINCREF(value); + Py_XINCREF(tb); + /* First, check the traceback argument, replacing None with NULL. */ + if (tb == Py_None) { + Py_DECREF(tb); + tb = 0; + } + else if (tb != NULL && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto raise_error; + } + /* Next, replace a missing value with None */ + if (value == NULL) { + value = Py_None; + Py_INCREF(value); + } + #if PY_VERSION_HEX < 0x02050000 + if (!PyClass_Check(type)) + #else + if (!PyType_Check(type)) + #endif + { + /* Raising an instance. The value should be a dummy. */ + if (value != Py_None) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto raise_error; + } + /* Normalize to raise , */ + Py_DECREF(value); + value = type; + #if PY_VERSION_HEX < 0x02050000 + if (PyInstance_Check(type)) { + type = (PyObject*) ((PyInstanceObject*)type)->in_class; + Py_INCREF(type); + } + else { + type = 0; + PyErr_SetString(PyExc_TypeError, + "raise: exception must be an old-style class or instance"); + goto raise_error; + } + #else + type = (PyObject*) Py_TYPE(type); + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } + #endif + } + __Pyx_ErrRestore(type, value, tb); + return; +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(tb); + return; +} + +static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyThreadState *tstate = PyThreadState_GET(); + +#if PY_MAJOR_VERSION >= 3 + /* Note: this is a temporary work-around to prevent crashes in Python 3.0 */ + if ((tstate->exc_type != NULL) & (tstate->exc_type != Py_None)) { + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + PyErr_NormalizeException(&type, &value, &tb); + PyErr_NormalizeException(&tmp_type, &tmp_value, &tmp_tb); + tstate->exc_type = 0; + tstate->exc_value = 0; + tstate->exc_traceback = 0; + PyException_SetContext(value, tmp_value); + Py_DECREF(tmp_type); + Py_XDECREF(tmp_tb); + } +#endif + + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} + +static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb) { + PyThreadState *tstate = PyThreadState_GET(); + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +} + + +static INLINE int __Pyx_StrEq(const char *s1, const char *s2) { + while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } + return *s1 == *s2; +} + +static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject* x) { + if (sizeof(unsigned char) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(unsigned char)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (unsigned char)-1; + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned char"); + return (unsigned char)-1; + } + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned char"); + return (unsigned char)-1; + } + return (unsigned char)val; + } + return (unsigned char)__Pyx_PyInt_AsUnsignedLong(x); +} + +static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject* x) { + if (sizeof(unsigned short) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(unsigned short)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (unsigned short)-1; + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned short"); + return (unsigned short)-1; + } + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned short"); + return (unsigned short)-1; + } + return (unsigned short)val; + } + return (unsigned short)__Pyx_PyInt_AsUnsignedLong(x); +} + +static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject* x) { + if (sizeof(unsigned int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(unsigned int)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (unsigned int)-1; + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned int"); + return (unsigned int)-1; + } + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned int"); + return (unsigned int)-1; + } + return (unsigned int)val; + } + return (unsigned int)__Pyx_PyInt_AsUnsignedLong(x); +} + +static INLINE char __Pyx_PyInt_AsChar(PyObject* x) { + if (sizeof(char) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(char)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (char)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to char"); + return (char)-1; + } + return (char)val; + } + return (char)__Pyx_PyInt_AsLong(x); +} + +static INLINE short __Pyx_PyInt_AsShort(PyObject* x) { + if (sizeof(short) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(short)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (short)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to short"); + return (short)-1; + } + return (short)val; + } + return (short)__Pyx_PyInt_AsLong(x); +} + +static INLINE int __Pyx_PyInt_AsInt(PyObject* x) { + if (sizeof(int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(int)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (int)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int)-1; + } + return (int)val; + } + return (int)__Pyx_PyInt_AsLong(x); +} + +static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject* x) { + if (sizeof(signed char) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(signed char)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (signed char)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to signed char"); + return (signed char)-1; + } + return (signed char)val; + } + return (signed char)__Pyx_PyInt_AsSignedLong(x); +} + +static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject* x) { + if (sizeof(signed short) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(signed short)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (signed short)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to signed short"); + return (signed short)-1; + } + return (signed short)val; + } + return (signed short)__Pyx_PyInt_AsSignedLong(x); +} + +static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject* x) { + if (sizeof(signed int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(signed int)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (signed int)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to signed int"); + return (signed int)-1; + } + return (signed int)val; + } + return (signed int)__Pyx_PyInt_AsSignedLong(x); +} + +static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned long"); + return (unsigned long)-1; + } + return (unsigned long)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + if (unlikely(Py_SIZE(x) < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned long"); + return (unsigned long)-1; + } + return PyLong_AsUnsignedLong(x); + } else { + unsigned long val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (unsigned long)-1; + val = __Pyx_PyInt_AsUnsignedLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned PY_LONG_LONG"); + return (unsigned PY_LONG_LONG)-1; + } + return (unsigned PY_LONG_LONG)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + if (unlikely(Py_SIZE(x) < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned PY_LONG_LONG"); + return (unsigned PY_LONG_LONG)-1; + } + return PyLong_AsUnsignedLongLong(x); + } else { + unsigned PY_LONG_LONG val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (unsigned PY_LONG_LONG)-1; + val = __Pyx_PyInt_AsUnsignedLongLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE long __Pyx_PyInt_AsLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (long)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLong(x); + } else { + long val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (long)-1; + val = __Pyx_PyInt_AsLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (PY_LONG_LONG)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLongLong(x); + } else { + PY_LONG_LONG val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (PY_LONG_LONG)-1; + val = __Pyx_PyInt_AsLongLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (signed long)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLong(x); + } else { + signed long val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (signed long)-1; + val = __Pyx_PyInt_AsSignedLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (signed PY_LONG_LONG)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLongLong(x); + } else { + signed PY_LONG_LONG val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (signed PY_LONG_LONG)-1; + val = __Pyx_PyInt_AsSignedLongLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static void __Pyx_WriteUnraisable(const char *name) { + PyObject *old_exc, *old_val, *old_tb; + PyObject *ctx; + __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); + #if PY_MAJOR_VERSION < 3 + ctx = PyString_FromString(name); + #else + ctx = PyUnicode_FromString(name); + #endif + __Pyx_ErrRestore(old_exc, old_val, old_tb); + if (!ctx) { + PyErr_WriteUnraisable(Py_None); + } else { + PyErr_WriteUnraisable(ctx); + Py_DECREF(ctx); + } +} + +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" + +static void __Pyx_AddTraceback(const char *funcname) { + PyObject *py_srcfile = 0; + PyObject *py_funcname = 0; + PyObject *py_globals = 0; + PyObject *empty_string = 0; + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + + #if PY_MAJOR_VERSION < 3 + py_srcfile = PyString_FromString(__pyx_filename); + #else + py_srcfile = PyUnicode_FromString(__pyx_filename); + #endif + if (!py_srcfile) goto bad; + if (__pyx_clineno) { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); + #else + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); + #endif + } + else { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromString(funcname); + #else + py_funcname = PyUnicode_FromString(funcname); + #endif + } + if (!py_funcname) goto bad; + py_globals = PyModule_GetDict(__pyx_m); + if (!py_globals) goto bad; + #if PY_MAJOR_VERSION < 3 + empty_string = PyString_FromStringAndSize("", 0); + #else + empty_string = PyBytes_FromStringAndSize("", 0); + #endif + if (!empty_string) goto bad; + py_code = PyCode_New( + 0, /*int argcount,*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*int kwonlyargcount,*/ + #endif + 0, /*int nlocals,*/ + 0, /*int stacksize,*/ + 0, /*int flags,*/ + empty_string, /*PyObject *code,*/ + __pyx_empty_tuple, /*PyObject *consts,*/ + __pyx_empty_tuple, /*PyObject *names,*/ + __pyx_empty_tuple, /*PyObject *varnames,*/ + __pyx_empty_tuple, /*PyObject *freevars,*/ + __pyx_empty_tuple, /*PyObject *cellvars,*/ + py_srcfile, /*PyObject *filename,*/ + py_funcname, /*PyObject *name,*/ + __pyx_lineno, /*int firstlineno,*/ + empty_string /*PyObject *lnotab*/ + ); + if (!py_code) goto bad; + py_frame = PyFrame_New( + PyThreadState_GET(), /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + py_globals, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + py_frame->f_lineno = __pyx_lineno; + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_srcfile); + Py_XDECREF(py_funcname); + Py_XDECREF(empty_string); + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} + +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { + while (t->p) { + #if PY_MAJOR_VERSION < 3 + if (t->is_unicode && (!t->is_identifier)) { + *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); + } else if (t->intern) { + *t->p = PyString_InternFromString(t->s); + } else { + *t->p = PyString_FromStringAndSize(t->s, t->n - 1); + } + #else /* Python 3+ has unicode identifiers */ + if (t->is_identifier || (t->is_unicode && t->intern)) { + *t->p = PyUnicode_InternFromString(t->s); + } else if (t->is_unicode) { + *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } else { + *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + #endif + if (!*t->p) + return -1; + ++t; + } + return 0; +} + +/* Type Conversion Functions */ + +static INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + if (x == Py_True) return 1; + else if ((x == Py_False) | (x == Py_None)) return 0; + else return PyObject_IsTrue(x); +} + +static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { + PyNumberMethods *m; + const char *name = NULL; + PyObject *res = NULL; +#if PY_VERSION_HEX < 0x03000000 + if (PyInt_Check(x) || PyLong_Check(x)) +#else + if (PyLong_Check(x)) +#endif + return Py_INCREF(x), x; + m = Py_TYPE(x)->tp_as_number; +#if PY_VERSION_HEX < 0x03000000 + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Int(x); + } + else if (m && m->nb_long) { + name = "long"; + res = PyNumber_Long(x); + } +#else + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Long(x); + } +#endif + if (res) { +#if PY_VERSION_HEX < 0x03000000 + if (!PyInt_Check(res) && !PyLong_Check(res)) { +#else + if (!PyLong_Check(res)) { +#endif + PyErr_Format(PyExc_TypeError, + "__%s__ returned non-%s (type %.200s)", + name, name, Py_TYPE(res)->tp_name); + Py_DECREF(res); + return NULL; + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} + +static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject* x = PyNumber_Index(b); + if (!x) return -1; + ival = PyInt_AsSsize_t(x); + Py_DECREF(x); + return ival; +} + +static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { +#if PY_VERSION_HEX < 0x02050000 + if (ival <= LONG_MAX) + return PyInt_FromLong((long)ival); + else { + unsigned char *bytes = (unsigned char *) &ival; + int one = 1; int little = (int)*(unsigned char*)&one; + return _PyLong_FromByteArray(bytes, sizeof(size_t), little, 0); + } +#else + return PyInt_FromSize_t(ival); +#endif +} + +static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject* x) { + unsigned PY_LONG_LONG val = __Pyx_PyInt_AsUnsignedLongLong(x); + if (unlikely(val == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())) { + return (size_t)-1; + } else if (unlikely(val != (unsigned PY_LONG_LONG)(size_t)val)) { + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t)-1; + } + return (size_t)val; +} + + diff --git a/python/setup.py b/python/setup.py new file mode 100644 index 0000000..8bfdf82 --- /dev/null +++ b/python/setup.py @@ -0,0 +1,29 @@ +from distutils.core import setup, Extension + +version = '0.0.1' + +msgpack_mod = Extension('msgpack', sources=['msgpack.c']) + +desc = 'MessagePack serializer/desirializer.' +long_desc = desc + """ + +Python binding of MessagePack_. + +This package is under development. + +.. _MessagePack: http://msgpack.sourceforge.jp/ + +What's MessagePack? (from http://msgpack.sourceforge.jp/) + + MessagePack is a binary-based efficient data interchange format that is + focused on high performance. It is like JSON, but very fast and small. +""" + +setup(name='msgpack', + author='Naoki INADA', + author_email='songofacandy@gmail.com', + version=version, + ext_modules=[msgpack_mod], + description='The MessagePack serializer/desirializer.', + long_description=long_desc, + ) From abe2a99ad2d57ef38c96643e93e0a1bb33dee1c9 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 01:43:50 +0900 Subject: [PATCH 0140/1172] add cythoned source and setup script. --- msgpack | 1 + msgpack.c | 3231 +++++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 29 + 3 files changed, 3261 insertions(+) create mode 120000 msgpack create mode 100644 msgpack.c create mode 100644 setup.py diff --git a/msgpack b/msgpack new file mode 120000 index 0000000..430db49 --- /dev/null +++ b/msgpack @@ -0,0 +1 @@ +../msgpack \ No newline at end of file diff --git a/msgpack.c b/msgpack.c new file mode 100644 index 0000000..50ec6fc --- /dev/null +++ b/msgpack.c @@ -0,0 +1,3231 @@ +/* Generated by Cython 0.11.2 on Mon Jun 8 01:28:30 2009 */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include "structmember.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#if PY_VERSION_HEX < 0x02040000 + #define METH_COEXIST 0 + #define PyDict_CheckExact(op) (Py_TYPE(op) == &PyDict_Type) +#endif +#if PY_VERSION_HEX < 0x02050000 + typedef int Py_ssize_t; + #define PY_SSIZE_T_MAX INT_MAX + #define PY_SSIZE_T_MIN INT_MIN + #define PY_FORMAT_SIZE_T "" + #define PyInt_FromSsize_t(z) PyInt_FromLong(z) + #define PyInt_AsSsize_t(o) PyInt_AsLong(o) + #define PyNumber_Index(o) PyNumber_Int(o) + #define PyIndex_Check(o) PyNumber_Check(o) +#endif +#if PY_VERSION_HEX < 0x02060000 + #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) + #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) + #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) + #define PyVarObject_HEAD_INIT(type, size) \ + PyObject_HEAD_INIT(type) size, + #define PyType_Modified(t) + + typedef struct { + void *buf; + PyObject *obj; + Py_ssize_t len; + Py_ssize_t itemsize; + int readonly; + int ndim; + char *format; + Py_ssize_t *shape; + Py_ssize_t *strides; + Py_ssize_t *suboffsets; + void *internal; + } Py_buffer; + + #define PyBUF_SIMPLE 0 + #define PyBUF_WRITABLE 0x0001 + #define PyBUF_FORMAT 0x0004 + #define PyBUF_ND 0x0008 + #define PyBUF_STRIDES (0x0010 | PyBUF_ND) + #define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES) + #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES) + #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES) + #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES) + +#endif +#if PY_MAJOR_VERSION < 3 + #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" +#else + #define __Pyx_BUILTIN_MODULE_NAME "builtins" +#endif +#if PY_MAJOR_VERSION >= 3 + #define Py_TPFLAGS_CHECKTYPES 0 + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#if (PY_VERSION_HEX < 0x02060000) || (PY_MAJOR_VERSION >= 3) + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBaseString_Type PyUnicode_Type + #define PyString_Type PyBytes_Type + #define PyString_CheckExact PyBytes_CheckExact + #define PyInt_Type PyLong_Type + #define PyInt_Check(op) PyLong_Check(op) + #define PyInt_CheckExact(op) PyLong_CheckExact(op) + #define PyInt_FromString PyLong_FromString + #define PyInt_FromUnicode PyLong_FromUnicode + #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t + #define PyInt_FromSsize_t PyLong_FromSsize_t + #define PyInt_AsLong PyLong_AsLong + #define PyInt_AS_LONG PyLong_AS_LONG + #define PyInt_AsSsize_t PyLong_AsSsize_t + #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask + #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#else + #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) + #define PyBytes_Type PyString_Type +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyMethod_New(func, self, klass) PyInstanceMethod_New(func) +#endif +#if !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#else + #define _USE_MATH_DEFINES +#endif +#if PY_VERSION_HEX < 0x02050000 + #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),((char *)(n))) + #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),((char *)(n)),(a)) + #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),((char *)(n))) +#else + #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),(n)) + #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),(n),(a)) + #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),(n)) +#endif +#if PY_VERSION_HEX < 0x02050000 + #define __Pyx_NAMESTR(n) ((char *)(n)) + #define __Pyx_DOCSTR(n) ((char *)(n)) +#else + #define __Pyx_NAMESTR(n) (n) + #define __Pyx_DOCSTR(n) (n) +#endif +#ifdef __cplusplus +#define __PYX_EXTERN_C extern "C" +#else +#define __PYX_EXTERN_C extern +#endif +#include +#define __PYX_HAVE_API__msgpack +#include "stdlib.h" +#include "string.h" +#include "pack.h" +#include "unpack.h" +#define __PYX_USE_C99_COMPLEX defined(_Complex_I) + + +#ifdef __GNUC__ +#define INLINE __inline__ +#elif _WIN32 +#define INLINE __inline +#else +#define INLINE +#endif + +typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern; char is_identifier;} __Pyx_StringTabEntry; /*proto*/ + + + +static int __pyx_skip_dispatch = 0; + + +/* Type Conversion Predeclarations */ + +#if PY_MAJOR_VERSION < 3 +#define __Pyx_PyBytes_FromString PyString_FromString +#define __Pyx_PyBytes_FromStringAndSize PyString_FromStringAndSize +#define __Pyx_PyBytes_AsString PyString_AsString +#else +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +#define __Pyx_PyBytes_AsString PyBytes_AsString +#endif + +#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False)) +static INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x); + +#if !defined(T_PYSSIZET) +#if PY_VERSION_HEX < 0x02050000 +#define T_PYSSIZET T_INT +#elif !defined(T_LONGLONG) +#define T_PYSSIZET \ + ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ + ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : -1)) +#else +#define T_PYSSIZET \ + ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ + ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : \ + ((sizeof(Py_ssize_t) == sizeof(PY_LONG_LONG)) ? T_LONGLONG : -1))) +#endif +#endif + +#if !defined(T_SIZET) +#if !defined(T_ULONGLONG) +#define T_SIZET \ + ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ + ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : -1)) +#else +#define T_SIZET \ + ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ + ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : \ + ((sizeof(size_t) == sizeof(unsigned PY_LONG_LONG)) ? T_ULONGLONG : -1))) +#endif +#endif + +static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); +static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*); + +#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) + + +#ifdef __GNUC__ +/* Test for GCC > 2.95 */ +#if __GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)) +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else /* __GNUC__ > 2 ... */ +#define likely(x) (x) +#define unlikely(x) (x) +#endif /* __GNUC__ > 2 ... */ +#else /* __GNUC__ */ +#define likely(x) (x) +#define unlikely(x) (x) +#endif /* __GNUC__ */ + +static PyObject *__pyx_m; +static PyObject *__pyx_b; +static PyObject *__pyx_empty_tuple; +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * __pyx_cfilenm= __FILE__; +static const char *__pyx_filename; +static const char **__pyx_f; + + +#ifdef CYTHON_REFNANNY +typedef struct { + void (*INCREF)(void*, PyObject*, int); + void (*DECREF)(void*, PyObject*, int); + void (*GOTREF)(void*, PyObject*, int); + void (*GIVEREF)(void*, PyObject*, int); + void* (*NewContext)(const char*, int, const char*); + void (*FinishContext)(void**); +} __Pyx_RefnannyAPIStruct; +static __Pyx_RefnannyAPIStruct *__Pyx_Refnanny = NULL; +#define __Pyx_ImportRefcountAPI(name) (__Pyx_RefnannyAPIStruct *) PyCObject_Import((char *)name, (char *)"RefnannyAPI") +#define __Pyx_INCREF(r) __Pyx_Refnanny->INCREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_DECREF(r) __Pyx_Refnanny->DECREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_GOTREF(r) __Pyx_Refnanny->GOTREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_GIVEREF(r) __Pyx_Refnanny->GIVEREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_XDECREF(r) if((r) == NULL) ; else __Pyx_DECREF(r) +#define __Pyx_SetupRefcountContext(name) void* __pyx_refchk = __Pyx_Refnanny->NewContext((name), __LINE__, __FILE__) +#define __Pyx_FinishRefcountContext() __Pyx_Refnanny->FinishContext(&__pyx_refchk) +#else +#define __Pyx_INCREF(r) Py_INCREF(r) +#define __Pyx_DECREF(r) Py_DECREF(r) +#define __Pyx_GOTREF(r) +#define __Pyx_GIVEREF(r) +#define __Pyx_XDECREF(r) Py_XDECREF(r) +#define __Pyx_SetupRefcountContext(name) +#define __Pyx_FinishRefcountContext() +#endif /* CYTHON_REFNANNY */ +#define __Pyx_XGIVEREF(r) if((r) == NULL) ; else __Pyx_GIVEREF(r) +#define __Pyx_XGOTREF(r) if((r) == NULL) ; else __Pyx_GOTREF(r) + +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, PyObject* kw_name); /*proto*/ + +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); /*proto*/ + +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, const char* function_name); /*proto*/ + +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list); /*proto*/ + +static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name); /*proto*/ + +static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); + +static INLINE void __Pyx_RaiseTooManyValuesError(void); + +static PyObject *__Pyx_UnpackItem(PyObject *, Py_ssize_t index); /*proto*/ +static int __Pyx_EndUnpack(PyObject *); /*proto*/ + +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ + +static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ +static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ + +static INLINE int __Pyx_StrEq(const char *, const char *); /*proto*/ + +static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject *); + +static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject *); + +static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject *); + +static INLINE char __Pyx_PyInt_AsChar(PyObject *); + +static INLINE short __Pyx_PyInt_AsShort(PyObject *); + +static INLINE int __Pyx_PyInt_AsInt(PyObject *); + +static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject *); + +static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject *); + +static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject *); + +static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject *); + +static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject *); + +static INLINE long __Pyx_PyInt_AsLong(PyObject *); + +static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject *); + +static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject *); + +static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); + +static void __Pyx_WriteUnraisable(const char *name); /*proto*/ + +static void __Pyx_AddTraceback(const char *funcname); /*proto*/ + +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ + +/* Type declarations */ + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":39 + * cdef int BUFF_SIZE=2*1024 + * + * cdef class Packer: # <<<<<<<<<<<<<< + * """Packer that pack data into strm. + * + */ + +struct __pyx_obj_7msgpack_Packer { + PyObject_HEAD + char *buff; + unsigned int length; + unsigned int allocated; + struct msgpack_packer pk; + PyObject *strm; +}; + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":193 + * return unpacks(packed) + * + * cdef class Unpacker: # <<<<<<<<<<<<<< + * """Do nothing. This function is for symmetric to Packer""" + * unpack = staticmethod(unpacks) + */ + +struct __pyx_obj_7msgpack_Unpacker { + PyObject_HEAD +}; +/* Module declarations from msgpack */ + +static PyTypeObject *__pyx_ptype_7msgpack_Packer = 0; +static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; +static int __pyx_v_7msgpack_BUFF_SIZE; +static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ +#define __Pyx_MODULE_NAME "msgpack" +int __pyx_module_is_main_msgpack = 0; + +/* Implementation of msgpack */ +static char __pyx_k___main__[] = "__main__"; +static PyObject *__pyx_kp___main__; +static char __pyx_k___init__[] = "__init__"; +static PyObject *__pyx_kp___init__; +static char __pyx_k_flush[] = "flush"; +static PyObject *__pyx_kp_flush; +static char __pyx_k_pack_list[] = "pack_list"; +static PyObject *__pyx_kp_pack_list; +static char __pyx_k_pack_dict[] = "pack_dict"; +static PyObject *__pyx_kp_pack_dict; +static char __pyx_k_pack[] = "pack"; +static PyObject *__pyx_kp_pack; +static char __pyx_k_unpack[] = "unpack"; +static PyObject *__pyx_kp_unpack; +static char __pyx_k_strm[] = "strm"; +static PyObject *__pyx_kp_strm; +static char __pyx_k_size[] = "size"; +static PyObject *__pyx_kp_size; +static char __pyx_k_len[] = "len"; +static PyObject *__pyx_kp_len; +static char __pyx_k_o[] = "o"; +static PyObject *__pyx_kp_o; +static char __pyx_k_stream[] = "stream"; +static PyObject *__pyx_kp_stream; +static char __pyx_k_packed_bytes[] = "packed_bytes"; +static PyObject *__pyx_kp_packed_bytes; +static char __pyx_k_cStringIO[] = "cStringIO"; +static PyObject *__pyx_kp_cStringIO; +static char __pyx_k_StringIO[] = "StringIO"; +static PyObject *__pyx_kp_StringIO; +static char __pyx_k_staticmethod[] = "staticmethod"; +static PyObject *__pyx_kp_staticmethod; +static char __pyx_k_unpacks[] = "unpacks"; +static PyObject *__pyx_kp_unpacks; +static char __pyx_k_write[] = "write"; +static PyObject *__pyx_kp_write; +static char __pyx_k_1[] = "flush"; +static PyObject *__pyx_kp_1; +static char __pyx_k_encode[] = "encode"; +static PyObject *__pyx_kp_encode; +static char __pyx_k_iteritems[] = "iteritems"; +static PyObject *__pyx_kp_iteritems; +static char __pyx_k_TypeError[] = "TypeError"; +static PyObject *__pyx_kp_TypeError; +static char __pyx_k_getvalue[] = "getvalue"; +static PyObject *__pyx_kp_getvalue; +static char __pyx_k_read[] = "read"; +static PyObject *__pyx_kp_read; +static PyObject *__pyx_builtin_staticmethod; +static PyObject *__pyx_builtin_TypeError; +static PyObject *__pyx_kp_2; +static PyObject *__pyx_kp_3; +static char __pyx_k_2[] = "utf-8"; +static char __pyx_k_3[] = "can't serialize %r"; + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":51 + * cdef object strm + * + * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< + * if size <= 0: + * size = BUFF_SIZE + */ + +static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_strm = 0; + int __pyx_v_size; + int __pyx_r; + int __pyx_t_1; + static PyObject **__pyx_pyargnames[] = {&__pyx_kp_strm,&__pyx_kp_size,0}; + __Pyx_SetupRefcountContext("__init__"); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); + PyObject* values[2] = {0,0}; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_strm); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (kw_args > 1) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_size); + if (unlikely(value)) { values[1] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "__init__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + __pyx_v_strm = values[0]; + if (values[1]) { + __pyx_v_size = __Pyx_PyInt_AsInt(values[1]); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } else { + __pyx_v_size = 0; + } + } else { + __pyx_v_size = 0; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: __pyx_v_size = __Pyx_PyInt_AsInt(PyTuple_GET_ITEM(__pyx_args, 1)); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + case 1: __pyx_v_strm = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("msgpack.Packer.__init__"); + return -1; + __pyx_L4_argument_unpacking_done:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":52 + * + * def __init__(self, strm, int size=0): + * if size <= 0: # <<<<<<<<<<<<<< + * size = BUFF_SIZE + * + */ + __pyx_t_1 = (__pyx_v_size <= 0); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":53 + * def __init__(self, strm, int size=0): + * if size <= 0: + * size = BUFF_SIZE # <<<<<<<<<<<<<< + * + * self.strm = strm + */ + __pyx_v_size = __pyx_v_7msgpack_BUFF_SIZE; + goto __pyx_L6; + } + __pyx_L6:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":55 + * size = BUFF_SIZE + * + * self.strm = strm # <<<<<<<<<<<<<< + * self.buff = malloc(size) + * self.allocated = size + */ + __Pyx_INCREF(__pyx_v_strm); + __Pyx_GIVEREF(__pyx_v_strm); + __Pyx_GOTREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); + __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":56 + * + * self.strm = strm + * self.buff = malloc(size) # <<<<<<<<<<<<<< + * self.allocated = size + * self.length = 0 + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":57 + * self.strm = strm + * self.buff = malloc(size) + * self.allocated = size # <<<<<<<<<<<<<< + * self.length = 0 + * + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":58 + * self.buff = malloc(size) + * self.allocated = size + * self.length = 0 # <<<<<<<<<<<<<< + * + * msgpack_packer_init(&self.pk, self, _packer_write) + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":60 + * self.length = 0 + * + * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< + * + * + */ + msgpack_packer_init((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), ((void *)__pyx_v_self), ((int (*)(void *, const char*, unsigned int))__pyx_f_7msgpack__packer_write)); + + __pyx_r = 0; + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":63 + * + * + * def flush(self): # <<<<<<<<<<<<<< + * """Flash local buffer and output stream if it has 'flush()' method.""" + * if self.length > 0: + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ +static char __pyx_doc_7msgpack_6Packer_flush[] = "Flash local buffer and output stream if it has 'flush()' method."; +static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused) { + PyObject *__pyx_r = NULL; + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + __Pyx_SetupRefcountContext("flush"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":65 + * def flush(self): + * """Flash local buffer and output stream if it has 'flush()' method.""" + * if self.length > 0: # <<<<<<<<<<<<<< + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + * self.length = 0 + */ + __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":66 + * """Flash local buffer and output stream if it has 'flush()' method.""" + * if self.length > 0: + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< + * self.length = 0 + * if hasattr(self.strm, 'flush'): + */ + __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_4)); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":67 + * if self.length > 0: + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + * self.length = 0 # <<<<<<<<<<<<<< + * if hasattr(self.strm, 'flush'): + * self.strm.flush() + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; + goto __pyx_L5; + } + __pyx_L5:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":68 + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + * self.length = 0 + * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< + * self.strm.flush() + * + */ + __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_1); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":69 + * self.length = 0 + * if hasattr(self.strm, 'flush'): + * self.strm.flush() # <<<<<<<<<<<<<< + * + * def pack_list(self, len): + */ + __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + goto __pyx_L6; + } + __pyx_L6:; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("msgpack.Packer.flush"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":71 + * self.strm.flush() + * + * def pack_list(self, len): # <<<<<<<<<<<<<< + * """Start packing sequential objects. + * + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ +static char __pyx_doc_7msgpack_6Packer_pack_list[] = "Start packing sequential objects.\n\n Example:\n\n packer.pack_list(2)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack(['foo', 'bar'])\n "; +static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { + PyObject *__pyx_r = NULL; + size_t __pyx_t_1; + __Pyx_SetupRefcountContext("pack_list"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":84 + * packer.pack(['foo', 'bar']) + * """ + * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< + * + * def pack_dict(self, len): + */ + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("msgpack.Packer.pack_list"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":86 + * msgpack_pack_array(&self.pk, len) + * + * def pack_dict(self, len): # <<<<<<<<<<<<<< + * """Start packing key-value objects. + * + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ +static char __pyx_doc_7msgpack_6Packer_pack_dict[] = "Start packing key-value objects.\n\n Example:\n\n packer.pack_dict(1)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack({'foo', 'bar'})\n "; +static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { + PyObject *__pyx_r = NULL; + size_t __pyx_t_1; + __Pyx_SetupRefcountContext("pack_dict"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":99 + * packer.pack({'foo', 'bar'}) + * """ + * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< + * + * def pack(self, object o): + */ + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("msgpack.Packer.pack_dict"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":101 + * msgpack_pack_map(&self.pk, len) + * + * def pack(self, object o): # <<<<<<<<<<<<<< + * cdef long long intval + * cdef double fval + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o); /*proto*/ +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o) { + PY_LONG_LONG __pyx_v_intval; + double __pyx_v_fval; + char *__pyx_v_rawval; + PyObject *__pyx_v_k; + PyObject *__pyx_v_v; + PyObject *__pyx_r = NULL; + PyObject *__pyx_1 = 0; + PyObject *__pyx_2 = 0; + PyObject *__pyx_3 = 0; + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PY_LONG_LONG __pyx_t_3; + char *__pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + int __pyx_t_9; + int __pyx_t_10; + __Pyx_SetupRefcountContext("pack"); + __Pyx_INCREF(__pyx_v_o); + __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); + __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":106 + * cdef char* rawval + * + * if o is None: # <<<<<<<<<<<<<< + * msgpack_pack_nil(&self.pk) + * elif o is True: + */ + __pyx_t_1 = (__pyx_v_o == Py_None); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":107 + * + * if o is None: + * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< + * elif o is True: + * msgpack_pack_true(&self.pk) + */ + msgpack_pack_nil((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":108 + * if o is None: + * msgpack_pack_nil(&self.pk) + * elif o is True: # <<<<<<<<<<<<<< + * msgpack_pack_true(&self.pk) + * elif o is False: + */ + __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = (__pyx_v_o == __pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":109 + * msgpack_pack_nil(&self.pk) + * elif o is True: + * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< + * elif o is False: + * msgpack_pack_false(&self.pk) + */ + msgpack_pack_true((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":110 + * elif o is True: + * msgpack_pack_true(&self.pk) + * elif o is False: # <<<<<<<<<<<<<< + * msgpack_pack_false(&self.pk) + * elif isinstance(o, long): + */ + __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = (__pyx_v_o == __pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":111 + * msgpack_pack_true(&self.pk) + * elif o is False: + * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< + * elif isinstance(o, long): + * intval = o + */ + msgpack_pack_false((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":112 + * elif o is False: + * msgpack_pack_false(&self.pk) + * elif isinstance(o, long): # <<<<<<<<<<<<<< + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":113 + * msgpack_pack_false(&self.pk) + * elif isinstance(o, long): + * intval = o # <<<<<<<<<<<<<< + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, int): + */ + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_intval = __pyx_t_3; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":114 + * elif isinstance(o, long): + * intval = o + * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< + * elif isinstance(o, int): + * intval = o + */ + msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":115 + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, int): # <<<<<<<<<<<<<< + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":116 + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, int): + * intval = o # <<<<<<<<<<<<<< + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, float): + */ + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_intval = __pyx_t_3; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":117 + * elif isinstance(o, int): + * intval = o + * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< + * elif isinstance(o, float): + * fval = 9 + */ + msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":118 + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, float): # <<<<<<<<<<<<<< + * fval = 9 + * msgpack_pack_double(&self.pk, fval) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":119 + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, float): + * fval = 9 # <<<<<<<<<<<<<< + * msgpack_pack_double(&self.pk, fval) + * elif isinstance(o, str): + */ + __pyx_v_fval = 9; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":120 + * elif isinstance(o, float): + * fval = 9 + * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< + * elif isinstance(o, str): + * rawval = o + */ + msgpack_pack_double((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_fval); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":121 + * fval = 9 + * msgpack_pack_double(&self.pk, fval) + * elif isinstance(o, str): # <<<<<<<<<<<<<< + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":122 + * msgpack_pack_double(&self.pk, fval) + * elif isinstance(o, str): + * rawval = o # <<<<<<<<<<<<<< + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + */ + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_4; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":123 + * elif isinstance(o, str): + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, unicode): + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":124 + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< + * elif isinstance(o, unicode): + * o = o.encode('utf-8') + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":125 + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, unicode): # <<<<<<<<<<<<<< + * o = o.encode('utf-8') + * rawval = o + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":126 + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, unicode): + * o = o.encode('utf-8') # <<<<<<<<<<<<<< + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_6)); + __Pyx_INCREF(__pyx_kp_2); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_2); + __Pyx_GIVEREF(__pyx_kp_2); + __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_v_o); + __pyx_v_o = __pyx_t_7; + __pyx_t_7 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":127 + * elif isinstance(o, unicode): + * o = o.encode('utf-8') + * rawval = o # <<<<<<<<<<<<<< + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + */ + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_4; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":128 + * o = o.encode('utf-8') + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, dict): + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":129 + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< + * elif isinstance(o, dict): + * msgpack_pack_map(&self.pk, len(o)) + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":130 + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, dict): # <<<<<<<<<<<<<< + * msgpack_pack_map(&self.pk, len(o)) + * for k,v in o.iteritems(): + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":131 + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, dict): + * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< + * for k,v in o.iteritems(): + * self.pack(k) + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":132 + * elif isinstance(o, dict): + * msgpack_pack_map(&self.pk, len(o)) + * for k,v in o.iteritems(): # <<<<<<<<<<<<<< + * self.pack(k) + * self.pack(v) + */ + __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { + __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); + } else { + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + for (;;) { + if (likely(PyList_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else { + __pyx_t_6 = PyIter_Next(__pyx_t_7); + if (!__pyx_t_6) { + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + break; + } + __Pyx_GOTREF(__pyx_t_6); + } + if (PyTuple_CheckExact(__pyx_t_6) && likely(PyTuple_GET_SIZE(__pyx_t_6) == 2)) { + PyObject* tuple = __pyx_t_6; + __pyx_2 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_2); + __pyx_3 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_3); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_v_k); + __pyx_v_k = __pyx_2; + __pyx_2 = 0; + __Pyx_DECREF(__pyx_v_v); + __pyx_v_v = __pyx_3; + __pyx_3 = 0; + } else { + __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_2); + __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_3); + if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + __Pyx_DECREF(__pyx_v_k); + __pyx_v_k = __pyx_2; + __pyx_2 = 0; + __Pyx_DECREF(__pyx_v_v); + __pyx_v_v = __pyx_3; + __pyx_3 = 0; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":133 + * msgpack_pack_map(&self.pk, len(o)) + * for k,v in o.iteritems(): + * self.pack(k) # <<<<<<<<<<<<<< + * self.pack(v) + * elif isinstance(o, tuple) or isinstance(o, list): + */ + __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_k); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); + __Pyx_GIVEREF(__pyx_v_k); + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":134 + * for k,v in o.iteritems(): + * self.pack(k) + * self.pack(v) # <<<<<<<<<<<<<< + * elif isinstance(o, tuple) or isinstance(o, list): + * msgpack_pack_array(&self.pk, len(o)) + */ + __pyx_t_8 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_v); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); + __Pyx_GIVEREF(__pyx_v_v); + __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":135 + * self.pack(k) + * self.pack(v) + * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< + * msgpack_pack_array(&self.pk, len(o)) + * for v in o: + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyTuple_Type))); + if (!__pyx_t_1) { + __pyx_t_9 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); + __pyx_t_10 = __pyx_t_9; + } else { + __pyx_t_10 = __pyx_t_1; + } + if (__pyx_t_10) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":136 + * self.pack(v) + * elif isinstance(o, tuple) or isinstance(o, list): + * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< + * for v in o: + * self.pack(v) + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":137 + * elif isinstance(o, tuple) or isinstance(o, list): + * msgpack_pack_array(&self.pk, len(o)) + * for v in o: # <<<<<<<<<<<<<< + * self.pack(v) + * else: + */ + if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { + __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); + } else { + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + } + for (;;) { + if (likely(PyList_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else { + __pyx_t_6 = PyIter_Next(__pyx_t_7); + if (!__pyx_t_6) { + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + break; + } + __Pyx_GOTREF(__pyx_t_6); + } + __Pyx_DECREF(__pyx_v_v); + __pyx_v_v = __pyx_t_6; + __pyx_t_6 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":138 + * msgpack_pack_array(&self.pk, len(o)) + * for v in o: + * self.pack(v) # <<<<<<<<<<<<<< + * else: + * # TODO: Serialize with defalt() like simplejson. + */ + __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_v); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); + __Pyx_GIVEREF(__pyx_v_v); + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L5; + } + /*else*/ { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":141 + * else: + * # TODO: Serialize with defalt() like simplejson. + * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + */ + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_7)); + __Pyx_INCREF(__pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); + __Pyx_GIVEREF(__pyx_v_o); + __pyx_t_8 = PyNumber_Remainder(__pyx_kp_3, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; + __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } + __pyx_L5:; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_2); + __Pyx_XDECREF(__pyx_3); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("msgpack.Packer.pack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_k); + __Pyx_DECREF(__pyx_v_v); + __Pyx_DECREF(__pyx_v_o); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":143 + * raise TypeError, "can't serialize %r" % (o,) + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< + * if packer.length + l > packer.allocated: + * if packer.length > 0: + */ + +static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__pyx_v_packer, const char* __pyx_v_b, unsigned int __pyx_v_l) { + int __pyx_r; + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + __Pyx_SetupRefcountContext("_packer_write"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":144 + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< + * if packer.length > 0: + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + */ + __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":145 + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + * if packer.length + l > packer.allocated: + * if packer.length > 0: # <<<<<<<<<<<<<< + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + * if l > 64: + */ + __pyx_t_1 = (__pyx_v_packer->length > 0); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":146 + * if packer.length + l > packer.allocated: + * if packer.length > 0: + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< + * if l > 64: + * packer.strm.write(PyString_FromStringAndSize(b, l)) + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_4)); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L4; + } + __pyx_L4:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":147 + * if packer.length > 0: + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + * if l > 64: # <<<<<<<<<<<<<< + * packer.strm.write(PyString_FromStringAndSize(b, l)) + * packer.length = 0 + */ + __pyx_t_1 = (__pyx_v_l > 64); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":148 + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + * if l > 64: + * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< + * packer.length = 0 + * else: + */ + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":149 + * if l > 64: + * packer.strm.write(PyString_FromStringAndSize(b, l)) + * packer.length = 0 # <<<<<<<<<<<<<< + * else: + * memcpy(packer.buff, b, l) + */ + __pyx_v_packer->length = 0; + goto __pyx_L5; + } + /*else*/ { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":151 + * packer.length = 0 + * else: + * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< + * packer.length = l + * else: + */ + memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":152 + * else: + * memcpy(packer.buff, b, l) + * packer.length = l # <<<<<<<<<<<<<< + * else: + * memcpy(packer.buff + packer.length, b, l) + */ + __pyx_v_packer->length = __pyx_v_l; + } + __pyx_L5:; + goto __pyx_L3; + } + /*else*/ { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":154 + * packer.length = l + * else: + * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< + * packer.length += l + * return 0 + */ + memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":155 + * else: + * memcpy(packer.buff + packer.length, b, l) + * packer.length += l # <<<<<<<<<<<<<< + * return 0 + * + */ + __pyx_v_packer->length += __pyx_v_l; + } + __pyx_L3:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":156 + * memcpy(packer.buff + packer.length, b, l) + * packer.length += l + * return 0 # <<<<<<<<<<<<<< + * + * def pack(object o, object stream): + */ + __pyx_r = 0; + goto __pyx_L0; + + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_WriteUnraisable("msgpack._packer_write"); + __pyx_r = 0; + __pyx_L0:; + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":158 + * return 0 + * + * def pack(object o, object stream): # <<<<<<<<<<<<<< + * packer = Packer(stream) + * packer.pack(o) + */ + +static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_o = 0; + PyObject *__pyx_v_stream = 0; + PyObject *__pyx_v_packer; + PyObject *__pyx_r = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + static PyObject **__pyx_pyargnames[] = {&__pyx_kp_o,&__pyx_kp_stream,0}; + __Pyx_SetupRefcountContext("pack"); + __pyx_self = __pyx_self; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); + PyObject* values[2] = {0,0}; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_o); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + values[1] = PyDict_GetItem(__pyx_kwds, __pyx_kp_stream); + if (likely(values[1])) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + __pyx_v_o = values[0]; + __pyx_v_stream = values[1]; + } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { + goto __pyx_L5_argtuple_error; + } else { + __pyx_v_o = PyTuple_GET_ITEM(__pyx_args, 0); + __pyx_v_stream = PyTuple_GET_ITEM(__pyx_args, 1); + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("msgpack.pack"); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":159 + * + * def pack(object o, object stream): + * packer = Packer(stream) # <<<<<<<<<<<<<< + * packer.pack(o) + * packer.flush() + */ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_stream); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_stream); + __Pyx_GIVEREF(__pyx_v_stream); + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_v_packer); + __pyx_v_packer = __pyx_t_2; + __pyx_t_2 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":160 + * def pack(object o, object stream): + * packer = Packer(stream) + * packer.pack(o) # <<<<<<<<<<<<<< + * packer.flush() + * + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); + __Pyx_GIVEREF(__pyx_v_o); + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":161 + * packer = Packer(stream) + * packer.pack(o) + * packer.flush() # <<<<<<<<<<<<<< + * + * def packs(object o): + */ + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("msgpack.pack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_packer); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":163 + * packer.flush() + * + * def packs(object o): # <<<<<<<<<<<<<< + * buf = StringIO() + * packer = Packer(buf) + */ + +static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o); /*proto*/ +static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o) { + PyObject *__pyx_v_buf; + PyObject *__pyx_v_packer; + PyObject *__pyx_r = NULL; + PyObject *__pyx_1 = 0; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + __Pyx_SetupRefcountContext("packs"); + __pyx_self = __pyx_self; + __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); + __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":164 + * + * def packs(object o): + * buf = StringIO() # <<<<<<<<<<<<<< + * packer = Packer(buf) + * packer.pack(o) + */ + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + __Pyx_DECREF(__pyx_v_buf); + __pyx_v_buf = __pyx_t_1; + __pyx_t_1 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":165 + * def packs(object o): + * buf = StringIO() + * packer = Packer(buf) # <<<<<<<<<<<<<< + * packer.pack(o) + * packer.flush() + */ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_buf); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_buf); + __Pyx_GIVEREF(__pyx_v_buf); + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_v_packer); + __pyx_v_packer = __pyx_t_2; + __pyx_t_2 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":166 + * buf = StringIO() + * packer = Packer(buf) + * packer.pack(o) # <<<<<<<<<<<<<< + * packer.flush() + * return buf.getvalue() + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); + __Pyx_GIVEREF(__pyx_v_o); + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":167 + * packer = Packer(buf) + * packer.pack(o) + * packer.flush() # <<<<<<<<<<<<<< + * return buf.getvalue() + * + */ + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":168 + * packer.pack(o) + * packer.flush() + * return buf.getvalue() # <<<<<<<<<<<<<< + * + * cdef extern from "unpack.h": + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("msgpack.packs"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_buf); + __Pyx_DECREF(__pyx_v_packer); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":179 + * + * + * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< + * """Unpack packed_bytes to object. Returns unpacked object.""" + * cdef const_char_ptr p = packed_bytes + */ + +static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes); /*proto*/ +static char __pyx_doc_7msgpack_unpacks[] = "Unpack packed_bytes to object. Returns unpacked object."; +static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes) { + const char* __pyx_v_p; + template_context __pyx_v_ctx; + size_t __pyx_v_off; + PyObject *__pyx_r = NULL; + const char* __pyx_t_1; + Py_ssize_t __pyx_t_2; + PyObject *__pyx_t_3; + __Pyx_SetupRefcountContext("unpacks"); + __pyx_self = __pyx_self; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":181 + * def unpacks(object packed_bytes): + * """Unpack packed_bytes to object. Returns unpacked object.""" + * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< + * cdef template_context ctx + * cdef size_t off = 0 + */ + __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 181; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_p = __pyx_t_1; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":183 + * cdef const_char_ptr p = packed_bytes + * cdef template_context ctx + * cdef size_t off = 0 # <<<<<<<<<<<<<< + * template_init(&ctx) + * template_execute(&ctx, p, len(packed_bytes), &off) + */ + __pyx_v_off = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":184 + * cdef template_context ctx + * cdef size_t off = 0 + * template_init(&ctx) # <<<<<<<<<<<<<< + * template_execute(&ctx, p, len(packed_bytes), &off) + * return template_data(&ctx) + */ + template_init((&__pyx_v_ctx)); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":185 + * cdef size_t off = 0 + * template_init(&ctx) + * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< + * return template_data(&ctx) + * + */ + __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 185; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":186 + * template_init(&ctx) + * template_execute(&ctx, p, len(packed_bytes), &off) + * return template_data(&ctx) # <<<<<<<<<<<<<< + * + * def unpack(object stream): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_3 = template_data((&__pyx_v_ctx)); + __Pyx_INCREF(((PyObject *)__pyx_t_3)); + __pyx_r = ((PyObject *)__pyx_t_3); + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("msgpack.unpacks"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":188 + * return template_data(&ctx) + * + * def unpack(object stream): # <<<<<<<<<<<<<< + * """unpack from stream.""" + * packed = stream.read() + */ + +static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream); /*proto*/ +static char __pyx_doc_7msgpack_unpack[] = "unpack from stream."; +static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream) { + PyObject *__pyx_v_packed; + PyObject *__pyx_r = NULL; + PyObject *__pyx_1 = 0; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + __Pyx_SetupRefcountContext("unpack"); + __pyx_self = __pyx_self; + __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":190 + * def unpack(object stream): + * """unpack from stream.""" + * packed = stream.read() # <<<<<<<<<<<<<< + * return unpacks(packed) + * + */ + __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_v_packed); + __pyx_v_packed = __pyx_t_2; + __pyx_t_2 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":191 + * """unpack from stream.""" + * packed = stream.read() + * return unpacks(packed) # <<<<<<<<<<<<<< + * + * cdef class Unpacker: + */ + __Pyx_XDECREF(__pyx_r); + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_packed); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_packed); + __Pyx_GIVEREF(__pyx_v_packed); + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("msgpack.unpack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_packed); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +static PyObject *__pyx_tp_new_7msgpack_Packer(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_7msgpack_Packer *p; + PyObject *o = (*t->tp_alloc)(t, 0); + if (!o) return 0; + p = ((struct __pyx_obj_7msgpack_Packer *)o); + p->strm = Py_None; Py_INCREF(Py_None); + return o; +} + +static void __pyx_tp_dealloc_7msgpack_Packer(PyObject *o) { + struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; + Py_XDECREF(p->strm); + (*Py_TYPE(o)->tp_free)(o); +} + +static int __pyx_tp_traverse_7msgpack_Packer(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; + if (p->strm) { + e = (*v)(p->strm, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_7msgpack_Packer(PyObject *o) { + struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; + PyObject* tmp; + tmp = ((PyObject*)p->strm); + p->strm = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static struct PyMethodDef __pyx_methods_7msgpack_Packer[] = { + {__Pyx_NAMESTR("flush"), (PyCFunction)__pyx_pf_7msgpack_6Packer_flush, METH_NOARGS, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_flush)}, + {__Pyx_NAMESTR("pack_list"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_list, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_list)}, + {__Pyx_NAMESTR("pack_dict"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_dict, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_dict)}, + {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_O, __Pyx_DOCSTR(0)}, + {0, 0, 0, 0} +}; + +static PyNumberMethods __pyx_tp_as_number_Packer = { + 0, /*nb_add*/ + 0, /*nb_subtract*/ + 0, /*nb_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_divide*/ + #endif + 0, /*nb_remainder*/ + 0, /*nb_divmod*/ + 0, /*nb_power*/ + 0, /*nb_negative*/ + 0, /*nb_positive*/ + 0, /*nb_absolute*/ + 0, /*nb_nonzero*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + 0, /*nb_and*/ + 0, /*nb_xor*/ + 0, /*nb_or*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_coerce*/ + #endif + 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else + 0, /*nb_long*/ + #endif + 0, /*nb_float*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_oct*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*nb_hex*/ + #endif + 0, /*nb_inplace_add*/ + 0, /*nb_inplace_subtract*/ + 0, /*nb_inplace_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_inplace_divide*/ + #endif + 0, /*nb_inplace_remainder*/ + 0, /*nb_inplace_power*/ + 0, /*nb_inplace_lshift*/ + 0, /*nb_inplace_rshift*/ + 0, /*nb_inplace_and*/ + 0, /*nb_inplace_xor*/ + 0, /*nb_inplace_or*/ + 0, /*nb_floor_divide*/ + 0, /*nb_true_divide*/ + 0, /*nb_inplace_floor_divide*/ + 0, /*nb_inplace_true_divide*/ + #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) + 0, /*nb_index*/ + #endif +}; + +static PySequenceMethods __pyx_tp_as_sequence_Packer = { + 0, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_Packer = { + 0, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_Packer = { + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getreadbuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getwritebuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getsegcount*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getcharbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_getbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_releasebuffer*/ + #endif +}; + +PyTypeObject __pyx_type_7msgpack_Packer = { + PyVarObject_HEAD_INIT(0, 0) + __Pyx_NAMESTR("msgpack.Packer"), /*tp_name*/ + sizeof(struct __pyx_obj_7msgpack_Packer), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_7msgpack_Packer, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + &__pyx_tp_as_number_Packer, /*tp_as_number*/ + &__pyx_tp_as_sequence_Packer, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_Packer, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_Packer, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __Pyx_DOCSTR("Packer that pack data into strm.\n\n strm must have `write(bytes)` method.\n size specifies local buffer size.\n "), /*tp_doc*/ + __pyx_tp_traverse_7msgpack_Packer, /*tp_traverse*/ + __pyx_tp_clear_7msgpack_Packer, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_7msgpack_Packer, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + __pyx_pf_7msgpack_6Packer___init__, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_7msgpack_Packer, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ +}; + +static PyObject *__pyx_tp_new_7msgpack_Unpacker(PyTypeObject *t, PyObject *a, PyObject *k) { + PyObject *o = (*t->tp_alloc)(t, 0); + if (!o) return 0; + return o; +} + +static void __pyx_tp_dealloc_7msgpack_Unpacker(PyObject *o) { + (*Py_TYPE(o)->tp_free)(o); +} + +static struct PyMethodDef __pyx_methods_7msgpack_Unpacker[] = { + {0, 0, 0, 0} +}; + +static PyNumberMethods __pyx_tp_as_number_Unpacker = { + 0, /*nb_add*/ + 0, /*nb_subtract*/ + 0, /*nb_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_divide*/ + #endif + 0, /*nb_remainder*/ + 0, /*nb_divmod*/ + 0, /*nb_power*/ + 0, /*nb_negative*/ + 0, /*nb_positive*/ + 0, /*nb_absolute*/ + 0, /*nb_nonzero*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + 0, /*nb_and*/ + 0, /*nb_xor*/ + 0, /*nb_or*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_coerce*/ + #endif + 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else + 0, /*nb_long*/ + #endif + 0, /*nb_float*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_oct*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*nb_hex*/ + #endif + 0, /*nb_inplace_add*/ + 0, /*nb_inplace_subtract*/ + 0, /*nb_inplace_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_inplace_divide*/ + #endif + 0, /*nb_inplace_remainder*/ + 0, /*nb_inplace_power*/ + 0, /*nb_inplace_lshift*/ + 0, /*nb_inplace_rshift*/ + 0, /*nb_inplace_and*/ + 0, /*nb_inplace_xor*/ + 0, /*nb_inplace_or*/ + 0, /*nb_floor_divide*/ + 0, /*nb_true_divide*/ + 0, /*nb_inplace_floor_divide*/ + 0, /*nb_inplace_true_divide*/ + #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) + 0, /*nb_index*/ + #endif +}; + +static PySequenceMethods __pyx_tp_as_sequence_Unpacker = { + 0, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_Unpacker = { + 0, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_Unpacker = { + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getreadbuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getwritebuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getsegcount*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getcharbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_getbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_releasebuffer*/ + #endif +}; + +PyTypeObject __pyx_type_7msgpack_Unpacker = { + PyVarObject_HEAD_INIT(0, 0) + __Pyx_NAMESTR("msgpack.Unpacker"), /*tp_name*/ + sizeof(struct __pyx_obj_7msgpack_Unpacker), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_7msgpack_Unpacker, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + &__pyx_tp_as_number_Unpacker, /*tp_as_number*/ + &__pyx_tp_as_sequence_Unpacker, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_Unpacker, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_Unpacker, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ + __Pyx_DOCSTR("Do nothing. This function is for symmetric to Packer"), /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_7msgpack_Unpacker, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_7msgpack_Unpacker, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ +}; + +static struct PyMethodDef __pyx_methods[] = { + {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("packs"), (PyCFunction)__pyx_pf_7msgpack_packs, METH_O, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("unpacks"), (PyCFunction)__pyx_pf_7msgpack_unpacks, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpacks)}, + {__Pyx_NAMESTR("unpack"), (PyCFunction)__pyx_pf_7msgpack_unpack, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpack)}, + {0, 0, 0, 0} +}; + +static void __pyx_init_filenames(void); /*proto*/ + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef __pyx_moduledef = { + PyModuleDef_HEAD_INIT, + __Pyx_NAMESTR("msgpack"), + 0, /* m_doc */ + -1, /* m_size */ + __pyx_methods /* m_methods */, + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ +}; +#endif + +static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, + {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, + {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, + {&__pyx_kp_pack_list, __pyx_k_pack_list, sizeof(__pyx_k_pack_list), 1, 1, 1}, + {&__pyx_kp_pack_dict, __pyx_k_pack_dict, sizeof(__pyx_k_pack_dict), 1, 1, 1}, + {&__pyx_kp_pack, __pyx_k_pack, sizeof(__pyx_k_pack), 1, 1, 1}, + {&__pyx_kp_unpack, __pyx_k_unpack, sizeof(__pyx_k_unpack), 1, 1, 1}, + {&__pyx_kp_strm, __pyx_k_strm, sizeof(__pyx_k_strm), 1, 1, 1}, + {&__pyx_kp_size, __pyx_k_size, sizeof(__pyx_k_size), 1, 1, 1}, + {&__pyx_kp_len, __pyx_k_len, sizeof(__pyx_k_len), 1, 1, 1}, + {&__pyx_kp_o, __pyx_k_o, sizeof(__pyx_k_o), 1, 1, 1}, + {&__pyx_kp_stream, __pyx_k_stream, sizeof(__pyx_k_stream), 1, 1, 1}, + {&__pyx_kp_packed_bytes, __pyx_k_packed_bytes, sizeof(__pyx_k_packed_bytes), 1, 1, 1}, + {&__pyx_kp_cStringIO, __pyx_k_cStringIO, sizeof(__pyx_k_cStringIO), 1, 1, 1}, + {&__pyx_kp_StringIO, __pyx_k_StringIO, sizeof(__pyx_k_StringIO), 1, 1, 1}, + {&__pyx_kp_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 1, 1, 1}, + {&__pyx_kp_unpacks, __pyx_k_unpacks, sizeof(__pyx_k_unpacks), 1, 1, 1}, + {&__pyx_kp_write, __pyx_k_write, sizeof(__pyx_k_write), 1, 1, 1}, + {&__pyx_kp_1, __pyx_k_1, sizeof(__pyx_k_1), 0, 1, 0}, + {&__pyx_kp_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 1, 1, 1}, + {&__pyx_kp_iteritems, __pyx_k_iteritems, sizeof(__pyx_k_iteritems), 1, 1, 1}, + {&__pyx_kp_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 1, 1, 1}, + {&__pyx_kp_getvalue, __pyx_k_getvalue, sizeof(__pyx_k_getvalue), 1, 1, 1}, + {&__pyx_kp_read, __pyx_k_read, sizeof(__pyx_k_read), 1, 1, 1}, + {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 0, 0}, + {&__pyx_kp_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 0}, + {0, 0, 0, 0, 0, 0} +}; +static int __Pyx_InitCachedBuiltins(void) { + __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + return 0; + __pyx_L1_error:; + return -1; +} + +static int __Pyx_InitGlobals(void) { + if (__Pyx_InitStrings(__pyx_string_tab) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + return 0; + __pyx_L1_error:; + return -1; +} + +#if PY_MAJOR_VERSION < 3 +PyMODINIT_FUNC initmsgpack(void); /*proto*/ +PyMODINIT_FUNC initmsgpack(void) +#else +PyMODINIT_FUNC PyInit_msgpack(void); /*proto*/ +PyMODINIT_FUNC PyInit_msgpack(void) +#endif +{ + PyObject *__pyx_1 = 0; + PyObject *__pyx_2 = 0; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + #ifdef CYTHON_REFNANNY + void* __pyx_refchk = NULL; + __Pyx_Refnanny = __Pyx_ImportRefcountAPI("refnanny"); + if (!__Pyx_Refnanny) { + PyErr_Clear(); + __Pyx_Refnanny = __Pyx_ImportRefcountAPI("Cython.Runtime.refnanny"); + if (!__Pyx_Refnanny) + Py_FatalError("failed to import refnanny module"); + } + __pyx_refchk = __Pyx_Refnanny->NewContext("PyMODINIT_FUNC PyInit_msgpack(void)", __LINE__, __FILE__); + #endif + __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + /*--- Library function declarations ---*/ + __pyx_init_filenames(); + /*--- Threads initialization code ---*/ + #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS + #ifdef WITH_THREAD /* Python build with threading support? */ + PyEval_InitThreads(); + #endif + #endif + /*--- Initialize various global constants etc. ---*/ + if (unlikely(__Pyx_InitGlobals() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + /*--- Module creation code ---*/ + #if PY_MAJOR_VERSION < 3 + __pyx_m = Py_InitModule4(__Pyx_NAMESTR("msgpack"), __pyx_methods, 0, 0, PYTHON_API_VERSION); + #else + __pyx_m = PyModule_Create(&__pyx_moduledef); + #endif + if (!__pyx_m) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + #if PY_MAJOR_VERSION < 3 + Py_INCREF(__pyx_m); + #endif + __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); + if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + if (__pyx_module_is_main_msgpack) { + if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + } + /*--- Builtin init code ---*/ + if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_skip_dispatch = 0; + /*--- Global init code ---*/ + /*--- Function export code ---*/ + /*--- Type init code ---*/ + if (PyType_Ready(&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "Packer", (PyObject *)&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_7msgpack_Packer = &__pyx_type_7msgpack_Packer; + if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_7msgpack_Unpacker = &__pyx_type_7msgpack_Unpacker; + /*--- Type import code ---*/ + /*--- Function import code ---*/ + /*--- Execution code ---*/ + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":3 + * # coding: utf-8 + * + * from cStringIO import StringIO # <<<<<<<<<<<<<< + * + * cdef extern from "Python.h": + */ + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_kp_StringIO); + PyList_SET_ITEM(__pyx_t_1, 0, __pyx_kp_StringIO); + __Pyx_GIVEREF(__pyx_kp_StringIO); + __pyx_1 = __Pyx_Import(__pyx_kp_cStringIO, ((PyObject *)__pyx_t_1)); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __pyx_2 = PyObject_GetAttr(__pyx_1, __pyx_kp_StringIO); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_2); + if (PyObject_SetAttr(__pyx_m, __pyx_kp_StringIO, __pyx_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_2); __pyx_2 = 0; + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":37 + * + * + * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< + * + * cdef class Packer: + */ + __pyx_v_7msgpack_BUFF_SIZE = 2048; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":195 + * cdef class Unpacker: + * """Do nothing. This function is for symmetric to Packer""" + * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< + */ + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_1); + __Pyx_GIVEREF(__pyx_1); + __pyx_1 = 0; + __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + PyType_Modified(__pyx_ptype_7msgpack_Unpacker); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_2); + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("msgpack"); + Py_DECREF(__pyx_m); __pyx_m = 0; + __pyx_L0:; + __Pyx_FinishRefcountContext(); + #if PY_MAJOR_VERSION < 3 + return; + #else + return __pyx_m; + #endif +} + +static const char *__pyx_filenames[] = { + "msgpack.pyx", +}; + +/* Runtime support code */ + +static void __pyx_init_filenames(void) { + __pyx_f = __pyx_filenames; +} + +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION >= 3 + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); + #else + "%s() got multiple values for keyword argument '%s'", func_name, + PyString_AS_STRING(kw_name)); + #endif +} + +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *number, *more_or_less; + + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + number = (num_expected == 1) ? "" : "s"; + PyErr_Format(PyExc_TypeError, + #if PY_VERSION_HEX < 0x02050000 + "%s() takes %s %d positional argument%s (%d given)", + #else + "%s() takes %s %zd positional argument%s (%zd given)", + #endif + func_name, more_or_less, num_expected, number, num_found); +} + +static int __Pyx_ParseOptionalKeywords( + PyObject *kwds, + PyObject **argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject *key = 0, *value = 0; + Py_ssize_t pos = 0; + PyObject*** name; + PyObject*** first_kw_arg = argnames + num_pos_args; + + while (PyDict_Next(kwds, &pos, &key, &value)) { + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + values[name-argnames] = value; + } else { + #if PY_MAJOR_VERSION < 3 + if (unlikely(!PyString_CheckExact(key)) && unlikely(!PyString_Check(key))) { + #else + if (unlikely(!PyUnicode_CheckExact(key)) && unlikely(!PyUnicode_Check(key))) { + #endif + goto invalid_keyword_type; + } else { + for (name = first_kw_arg; *name; name++) { + #if PY_MAJOR_VERSION >= 3 + if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && + PyUnicode_Compare(**name, key) == 0) break; + #else + if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && + _PyString_Eq(**name, key)) break; + #endif + } + if (*name) { + values[name-argnames] = value; + } else { + /* unexpected keyword found */ + for (name=argnames; name != first_kw_arg; name++) { + if (**name == key) goto arg_passed_twice; + #if PY_MAJOR_VERSION >= 3 + if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && + PyUnicode_Compare(**name, key) == 0) goto arg_passed_twice; + #else + if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && + _PyString_Eq(**name, key)) goto arg_passed_twice; + #endif + } + if (kwds2) { + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else { + goto invalid_keyword; + } + } + } + } + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, **name); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%s() keywords must be strings", function_name); + goto bad; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%s() got an unexpected keyword argument '%s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif +bad: + return -1; +} + +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list) { + PyObject *__import__ = 0; + PyObject *empty_list = 0; + PyObject *module = 0; + PyObject *global_dict = 0; + PyObject *empty_dict = 0; + PyObject *list; + __import__ = __Pyx_GetAttrString(__pyx_b, "__import__"); + if (!__import__) + goto bad; + if (from_list) + list = from_list; + else { + empty_list = PyList_New(0); + if (!empty_list) + goto bad; + list = empty_list; + } + global_dict = PyModule_GetDict(__pyx_m); + if (!global_dict) + goto bad; + empty_dict = PyDict_New(); + if (!empty_dict) + goto bad; + module = PyObject_CallFunctionObjArgs(__import__, + name, global_dict, empty_dict, list, NULL); +bad: + Py_XDECREF(empty_list); + Py_XDECREF(__import__); + Py_XDECREF(empty_dict); + return module; +} + +static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name) { + PyObject *result; + result = PyObject_GetAttr(dict, name); + if (!result) + PyErr_SetObject(PyExc_NameError, name); + return result; +} + +static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { + PyErr_Format(PyExc_ValueError, + #if PY_VERSION_HEX < 0x02050000 + "need more than %d value%s to unpack", (int)index, + #else + "need more than %zd value%s to unpack", index, + #endif + (index == 1) ? "" : "s"); +} + +static INLINE void __Pyx_RaiseTooManyValuesError(void) { + PyErr_SetString(PyExc_ValueError, "too many values to unpack"); +} + +static PyObject *__Pyx_UnpackItem(PyObject *iter, Py_ssize_t index) { + PyObject *item; + if (!(item = PyIter_Next(iter))) { + if (!PyErr_Occurred()) { + __Pyx_RaiseNeedMoreValuesError(index); + } + } + return item; +} + +static int __Pyx_EndUnpack(PyObject *iter) { + PyObject *item; + if ((item = PyIter_Next(iter))) { + Py_DECREF(item); + __Pyx_RaiseTooManyValuesError(); + return -1; + } + else if (!PyErr_Occurred()) + return 0; + else + return -1; +} + +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb) { + Py_XINCREF(type); + Py_XINCREF(value); + Py_XINCREF(tb); + /* First, check the traceback argument, replacing None with NULL. */ + if (tb == Py_None) { + Py_DECREF(tb); + tb = 0; + } + else if (tb != NULL && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto raise_error; + } + /* Next, replace a missing value with None */ + if (value == NULL) { + value = Py_None; + Py_INCREF(value); + } + #if PY_VERSION_HEX < 0x02050000 + if (!PyClass_Check(type)) + #else + if (!PyType_Check(type)) + #endif + { + /* Raising an instance. The value should be a dummy. */ + if (value != Py_None) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto raise_error; + } + /* Normalize to raise , */ + Py_DECREF(value); + value = type; + #if PY_VERSION_HEX < 0x02050000 + if (PyInstance_Check(type)) { + type = (PyObject*) ((PyInstanceObject*)type)->in_class; + Py_INCREF(type); + } + else { + type = 0; + PyErr_SetString(PyExc_TypeError, + "raise: exception must be an old-style class or instance"); + goto raise_error; + } + #else + type = (PyObject*) Py_TYPE(type); + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } + #endif + } + __Pyx_ErrRestore(type, value, tb); + return; +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(tb); + return; +} + +static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyThreadState *tstate = PyThreadState_GET(); + +#if PY_MAJOR_VERSION >= 3 + /* Note: this is a temporary work-around to prevent crashes in Python 3.0 */ + if ((tstate->exc_type != NULL) & (tstate->exc_type != Py_None)) { + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + PyErr_NormalizeException(&type, &value, &tb); + PyErr_NormalizeException(&tmp_type, &tmp_value, &tmp_tb); + tstate->exc_type = 0; + tstate->exc_value = 0; + tstate->exc_traceback = 0; + PyException_SetContext(value, tmp_value); + Py_DECREF(tmp_type); + Py_XDECREF(tmp_tb); + } +#endif + + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} + +static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb) { + PyThreadState *tstate = PyThreadState_GET(); + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +} + + +static INLINE int __Pyx_StrEq(const char *s1, const char *s2) { + while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } + return *s1 == *s2; +} + +static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject* x) { + if (sizeof(unsigned char) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(unsigned char)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (unsigned char)-1; + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned char"); + return (unsigned char)-1; + } + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned char"); + return (unsigned char)-1; + } + return (unsigned char)val; + } + return (unsigned char)__Pyx_PyInt_AsUnsignedLong(x); +} + +static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject* x) { + if (sizeof(unsigned short) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(unsigned short)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (unsigned short)-1; + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned short"); + return (unsigned short)-1; + } + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned short"); + return (unsigned short)-1; + } + return (unsigned short)val; + } + return (unsigned short)__Pyx_PyInt_AsUnsignedLong(x); +} + +static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject* x) { + if (sizeof(unsigned int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(unsigned int)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (unsigned int)-1; + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned int"); + return (unsigned int)-1; + } + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned int"); + return (unsigned int)-1; + } + return (unsigned int)val; + } + return (unsigned int)__Pyx_PyInt_AsUnsignedLong(x); +} + +static INLINE char __Pyx_PyInt_AsChar(PyObject* x) { + if (sizeof(char) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(char)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (char)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to char"); + return (char)-1; + } + return (char)val; + } + return (char)__Pyx_PyInt_AsLong(x); +} + +static INLINE short __Pyx_PyInt_AsShort(PyObject* x) { + if (sizeof(short) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(short)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (short)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to short"); + return (short)-1; + } + return (short)val; + } + return (short)__Pyx_PyInt_AsLong(x); +} + +static INLINE int __Pyx_PyInt_AsInt(PyObject* x) { + if (sizeof(int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(int)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (int)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int)-1; + } + return (int)val; + } + return (int)__Pyx_PyInt_AsLong(x); +} + +static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject* x) { + if (sizeof(signed char) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(signed char)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (signed char)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to signed char"); + return (signed char)-1; + } + return (signed char)val; + } + return (signed char)__Pyx_PyInt_AsSignedLong(x); +} + +static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject* x) { + if (sizeof(signed short) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(signed short)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (signed short)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to signed short"); + return (signed short)-1; + } + return (signed short)val; + } + return (signed short)__Pyx_PyInt_AsSignedLong(x); +} + +static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject* x) { + if (sizeof(signed int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(signed int)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (signed int)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to signed int"); + return (signed int)-1; + } + return (signed int)val; + } + return (signed int)__Pyx_PyInt_AsSignedLong(x); +} + +static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned long"); + return (unsigned long)-1; + } + return (unsigned long)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + if (unlikely(Py_SIZE(x) < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned long"); + return (unsigned long)-1; + } + return PyLong_AsUnsignedLong(x); + } else { + unsigned long val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (unsigned long)-1; + val = __Pyx_PyInt_AsUnsignedLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned PY_LONG_LONG"); + return (unsigned PY_LONG_LONG)-1; + } + return (unsigned PY_LONG_LONG)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + if (unlikely(Py_SIZE(x) < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned PY_LONG_LONG"); + return (unsigned PY_LONG_LONG)-1; + } + return PyLong_AsUnsignedLongLong(x); + } else { + unsigned PY_LONG_LONG val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (unsigned PY_LONG_LONG)-1; + val = __Pyx_PyInt_AsUnsignedLongLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE long __Pyx_PyInt_AsLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (long)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLong(x); + } else { + long val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (long)-1; + val = __Pyx_PyInt_AsLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (PY_LONG_LONG)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLongLong(x); + } else { + PY_LONG_LONG val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (PY_LONG_LONG)-1; + val = __Pyx_PyInt_AsLongLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (signed long)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLong(x); + } else { + signed long val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (signed long)-1; + val = __Pyx_PyInt_AsSignedLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (signed PY_LONG_LONG)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLongLong(x); + } else { + signed PY_LONG_LONG val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (signed PY_LONG_LONG)-1; + val = __Pyx_PyInt_AsSignedLongLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static void __Pyx_WriteUnraisable(const char *name) { + PyObject *old_exc, *old_val, *old_tb; + PyObject *ctx; + __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); + #if PY_MAJOR_VERSION < 3 + ctx = PyString_FromString(name); + #else + ctx = PyUnicode_FromString(name); + #endif + __Pyx_ErrRestore(old_exc, old_val, old_tb); + if (!ctx) { + PyErr_WriteUnraisable(Py_None); + } else { + PyErr_WriteUnraisable(ctx); + Py_DECREF(ctx); + } +} + +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" + +static void __Pyx_AddTraceback(const char *funcname) { + PyObject *py_srcfile = 0; + PyObject *py_funcname = 0; + PyObject *py_globals = 0; + PyObject *empty_string = 0; + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + + #if PY_MAJOR_VERSION < 3 + py_srcfile = PyString_FromString(__pyx_filename); + #else + py_srcfile = PyUnicode_FromString(__pyx_filename); + #endif + if (!py_srcfile) goto bad; + if (__pyx_clineno) { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); + #else + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); + #endif + } + else { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromString(funcname); + #else + py_funcname = PyUnicode_FromString(funcname); + #endif + } + if (!py_funcname) goto bad; + py_globals = PyModule_GetDict(__pyx_m); + if (!py_globals) goto bad; + #if PY_MAJOR_VERSION < 3 + empty_string = PyString_FromStringAndSize("", 0); + #else + empty_string = PyBytes_FromStringAndSize("", 0); + #endif + if (!empty_string) goto bad; + py_code = PyCode_New( + 0, /*int argcount,*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*int kwonlyargcount,*/ + #endif + 0, /*int nlocals,*/ + 0, /*int stacksize,*/ + 0, /*int flags,*/ + empty_string, /*PyObject *code,*/ + __pyx_empty_tuple, /*PyObject *consts,*/ + __pyx_empty_tuple, /*PyObject *names,*/ + __pyx_empty_tuple, /*PyObject *varnames,*/ + __pyx_empty_tuple, /*PyObject *freevars,*/ + __pyx_empty_tuple, /*PyObject *cellvars,*/ + py_srcfile, /*PyObject *filename,*/ + py_funcname, /*PyObject *name,*/ + __pyx_lineno, /*int firstlineno,*/ + empty_string /*PyObject *lnotab*/ + ); + if (!py_code) goto bad; + py_frame = PyFrame_New( + PyThreadState_GET(), /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + py_globals, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + py_frame->f_lineno = __pyx_lineno; + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_srcfile); + Py_XDECREF(py_funcname); + Py_XDECREF(empty_string); + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} + +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { + while (t->p) { + #if PY_MAJOR_VERSION < 3 + if (t->is_unicode && (!t->is_identifier)) { + *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); + } else if (t->intern) { + *t->p = PyString_InternFromString(t->s); + } else { + *t->p = PyString_FromStringAndSize(t->s, t->n - 1); + } + #else /* Python 3+ has unicode identifiers */ + if (t->is_identifier || (t->is_unicode && t->intern)) { + *t->p = PyUnicode_InternFromString(t->s); + } else if (t->is_unicode) { + *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } else { + *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + #endif + if (!*t->p) + return -1; + ++t; + } + return 0; +} + +/* Type Conversion Functions */ + +static INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + if (x == Py_True) return 1; + else if ((x == Py_False) | (x == Py_None)) return 0; + else return PyObject_IsTrue(x); +} + +static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { + PyNumberMethods *m; + const char *name = NULL; + PyObject *res = NULL; +#if PY_VERSION_HEX < 0x03000000 + if (PyInt_Check(x) || PyLong_Check(x)) +#else + if (PyLong_Check(x)) +#endif + return Py_INCREF(x), x; + m = Py_TYPE(x)->tp_as_number; +#if PY_VERSION_HEX < 0x03000000 + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Int(x); + } + else if (m && m->nb_long) { + name = "long"; + res = PyNumber_Long(x); + } +#else + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Long(x); + } +#endif + if (res) { +#if PY_VERSION_HEX < 0x03000000 + if (!PyInt_Check(res) && !PyLong_Check(res)) { +#else + if (!PyLong_Check(res)) { +#endif + PyErr_Format(PyExc_TypeError, + "__%s__ returned non-%s (type %.200s)", + name, name, Py_TYPE(res)->tp_name); + Py_DECREF(res); + return NULL; + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} + +static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject* x = PyNumber_Index(b); + if (!x) return -1; + ival = PyInt_AsSsize_t(x); + Py_DECREF(x); + return ival; +} + +static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { +#if PY_VERSION_HEX < 0x02050000 + if (ival <= LONG_MAX) + return PyInt_FromLong((long)ival); + else { + unsigned char *bytes = (unsigned char *) &ival; + int one = 1; int little = (int)*(unsigned char*)&one; + return _PyLong_FromByteArray(bytes, sizeof(size_t), little, 0); + } +#else + return PyInt_FromSize_t(ival); +#endif +} + +static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject* x) { + unsigned PY_LONG_LONG val = __Pyx_PyInt_AsUnsignedLongLong(x); + if (unlikely(val == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())) { + return (size_t)-1; + } else if (unlikely(val != (unsigned PY_LONG_LONG)(size_t)val)) { + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t)-1; + } + return (size_t)val; +} + + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..8bfdf82 --- /dev/null +++ b/setup.py @@ -0,0 +1,29 @@ +from distutils.core import setup, Extension + +version = '0.0.1' + +msgpack_mod = Extension('msgpack', sources=['msgpack.c']) + +desc = 'MessagePack serializer/desirializer.' +long_desc = desc + """ + +Python binding of MessagePack_. + +This package is under development. + +.. _MessagePack: http://msgpack.sourceforge.jp/ + +What's MessagePack? (from http://msgpack.sourceforge.jp/) + + MessagePack is a binary-based efficient data interchange format that is + focused on high performance. It is like JSON, but very fast and small. +""" + +setup(name='msgpack', + author='Naoki INADA', + author_email='songofacandy@gmail.com', + version=version, + ext_modules=[msgpack_mod], + description='The MessagePack serializer/desirializer.', + long_description=long_desc, + ) From 17d2ca2d63b3eb8b9d6028c180dc00f1f10be51e Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 04:33:47 +0900 Subject: [PATCH 0141/1172] Fix unpacking True, False and True. --- python/unpack.h | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/python/unpack.h b/python/unpack.h index eb72001..e51557f 100644 --- a/python/unpack.h +++ b/python/unpack.h @@ -26,7 +26,7 @@ typedef struct { struct template ## name #define msgpack_unpack_func(ret, name) \ - ret template ## name + static inline ret template ## name #define msgpack_unpack_callback(name) \ template_callback ## name @@ -39,16 +39,8 @@ typedef struct { struct template_context; typedef struct template_context template_context; -static inline void template_init(template_context* ctx); - -static inline msgpack_unpack_object template_data(template_context* ctx); - -static inline int template_execute(template_context* ctx, - const char* data, size_t len, size_t* off); - - static inline msgpack_unpack_object template_callback_root(unpack_user* u) -{ PyObject *o = Py_None; Py_INCREF(o); return o; } +{ return NULL; } static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) { *o = PyInt_FromLong((long)d); return 0; } @@ -88,13 +80,13 @@ static inline int template_callback_double(unpack_user* u, double d, msgpack_unp { *o = PyFloat_FromDouble(d); return 0; } static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) -{ *o = Py_None; Py_INCREF(o); return 0; } +{ Py_INCREF(Py_None); *o = Py_None; return 0; } static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o) -{ *o = Py_True; Py_INCREF(o); return 0; } +{ Py_INCREF(Py_True); *o = Py_True; return 0; } static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o) -{ *o = Py_False; Py_INCREF(o); return 0; } +{ Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { From 9b5a25ec32f716715311f2d2386762a1fd5714b7 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 04:33:47 +0900 Subject: [PATCH 0142/1172] Fix unpacking True, False and True. --- unpack.h | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/unpack.h b/unpack.h index eb72001..e51557f 100644 --- a/unpack.h +++ b/unpack.h @@ -26,7 +26,7 @@ typedef struct { struct template ## name #define msgpack_unpack_func(ret, name) \ - ret template ## name + static inline ret template ## name #define msgpack_unpack_callback(name) \ template_callback ## name @@ -39,16 +39,8 @@ typedef struct { struct template_context; typedef struct template_context template_context; -static inline void template_init(template_context* ctx); - -static inline msgpack_unpack_object template_data(template_context* ctx); - -static inline int template_execute(template_context* ctx, - const char* data, size_t len, size_t* off); - - static inline msgpack_unpack_object template_callback_root(unpack_user* u) -{ PyObject *o = Py_None; Py_INCREF(o); return o; } +{ return NULL; } static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) { *o = PyInt_FromLong((long)d); return 0; } @@ -88,13 +80,13 @@ static inline int template_callback_double(unpack_user* u, double d, msgpack_unp { *o = PyFloat_FromDouble(d); return 0; } static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) -{ *o = Py_None; Py_INCREF(o); return 0; } +{ Py_INCREF(Py_None); *o = Py_None; return 0; } static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o) -{ *o = Py_True; Py_INCREF(o); return 0; } +{ Py_INCREF(Py_True); *o = Py_True; return 0; } static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o) -{ *o = Py_False; Py_INCREF(o); return 0; } +{ Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { From 935db853f0d70576c84f908c69d809877796503b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 04:33:58 +0900 Subject: [PATCH 0143/1172] add test. --- python/testformat.py | 64 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 python/testformat.py diff --git a/python/testformat.py b/python/testformat.py new file mode 100644 index 0000000..a00123c --- /dev/null +++ b/python/testformat.py @@ -0,0 +1,64 @@ +from unittest import TestCase, main +from msgpack import packs, unpacks + +class TestFormat(TestCase): + def __check(self, obj, expected_packed): + packed = packs(obj) + self.assertEqual(packed, expected_packed) + unpacked = unpacks(packed) + self.assertEqual(unpacked, obj) + + def testSimpleValues(self): + self.__check(None, '\xc0') + self.__check(True, '\xc3') + self.__check(False, '\xc2') + self.__check( + [None, False, True], + '\x93\xc0\xc2\xc3' + ) + + def testFixnum(self): + self.__check( + [[0,64,127], [-32,-16,-1]], + "\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff" + ) + + def testFixArray(self): + self.__check( + [[],[[None]]], + "\x92\x90\x91\x91\xc0" + ) + + def testFixRaw(self): + self.__check( + ["", "a", "bc", "def"], + "\x94\xa0\xa1a\xa2bc\xa3def" + ) + pass + + def testFixMap(self): + self.__check( + {False: {None: None}, True:{None:{}}}, + "\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80" + ) + pass + + +class TestUnpack(TestCase): + def __check(self, packed, obj): + self.assertEqual(unpacks(packed), obj) + + def testuint(self): + self.__check( + "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" + "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" + "\xce\xff\xff\xff\xff", + [0, 128, 255, 0, 32768, 65535, 0, + 2147483648, 4294967295], + ) + + + + +if __name__ == '__main__': + main() From b0f42ca49eb4ad4770efe6587b7fe3b6f0e06da1 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 04:33:58 +0900 Subject: [PATCH 0144/1172] add test. --- testformat.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 testformat.py diff --git a/testformat.py b/testformat.py new file mode 100644 index 0000000..a00123c --- /dev/null +++ b/testformat.py @@ -0,0 +1,64 @@ +from unittest import TestCase, main +from msgpack import packs, unpacks + +class TestFormat(TestCase): + def __check(self, obj, expected_packed): + packed = packs(obj) + self.assertEqual(packed, expected_packed) + unpacked = unpacks(packed) + self.assertEqual(unpacked, obj) + + def testSimpleValues(self): + self.__check(None, '\xc0') + self.__check(True, '\xc3') + self.__check(False, '\xc2') + self.__check( + [None, False, True], + '\x93\xc0\xc2\xc3' + ) + + def testFixnum(self): + self.__check( + [[0,64,127], [-32,-16,-1]], + "\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff" + ) + + def testFixArray(self): + self.__check( + [[],[[None]]], + "\x92\x90\x91\x91\xc0" + ) + + def testFixRaw(self): + self.__check( + ["", "a", "bc", "def"], + "\x94\xa0\xa1a\xa2bc\xa3def" + ) + pass + + def testFixMap(self): + self.__check( + {False: {None: None}, True:{None:{}}}, + "\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80" + ) + pass + + +class TestUnpack(TestCase): + def __check(self, packed, obj): + self.assertEqual(unpacks(packed), obj) + + def testuint(self): + self.__check( + "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" + "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" + "\xce\xff\xff\xff\xff", + [0, 128, 255, 0, 32768, 65535, 0, + 2147483648, 4294967295], + ) + + + + +if __name__ == '__main__': + main() From 560bd901f83c9024ed88b738b80e14de337847c5 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 12:46:02 +0900 Subject: [PATCH 0145/1172] Fix double INCREF-ing when unpacking. --- python/msgpack.c | 20 +++++++++++--------- python/msgpack.pyx | 4 ++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/python/msgpack.c b/python/msgpack.c index 50ec6fc..e044f18 100644 --- a/python/msgpack.c +++ b/python/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Mon Jun 8 01:28:30 2009 */ +/* Generated by Cython 0.11.2 on Mon Jun 8 12:41:02 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -1724,7 +1724,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx PyObject *__pyx_r = NULL; const char* __pyx_t_1; Py_ssize_t __pyx_t_2; - PyObject *__pyx_t_3; + PyObject *__pyx_t_3 = NULL; __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; @@ -1752,7 +1752,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) + * return template_data(&ctx) */ template_init((&__pyx_v_ctx)); @@ -1760,7 +1760,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< - * return template_data(&ctx) + * return template_data(&ctx) * */ __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 185; __pyx_clineno = __LINE__; goto __pyx_L1_error;} @@ -1769,19 +1769,21 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":186 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) # <<<<<<<<<<<<<< + * return template_data(&ctx) # <<<<<<<<<<<<<< * * def unpack(object stream): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = template_data((&__pyx_v_ctx)); - __Pyx_INCREF(((PyObject *)__pyx_t_3)); - __pyx_r = ((PyObject *)__pyx_t_3); + __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 186; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); __Pyx_AddTraceback("msgpack.unpacks"); __pyx_r = NULL; __pyx_L0:; @@ -1791,7 +1793,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx } /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":188 - * return template_data(&ctx) + * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< * """unpack from stream.""" diff --git a/python/msgpack.pyx b/python/msgpack.pyx index c454c5d..8b2006a 100644 --- a/python/msgpack.pyx +++ b/python/msgpack.pyx @@ -173,7 +173,7 @@ cdef extern from "unpack.h": int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) void template_init(template_context* ctx) - PyObject* template_data(template_context* ctx) + object template_data(template_context* ctx) def unpacks(object packed_bytes): @@ -183,7 +183,7 @@ def unpacks(object packed_bytes): cdef size_t off = 0 template_init(&ctx) template_execute(&ctx, p, len(packed_bytes), &off) - return template_data(&ctx) + return template_data(&ctx) def unpack(object stream): """unpack from stream.""" From 46fae3d893be1fd75195c71b1503352f02d7ef38 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 12:46:02 +0900 Subject: [PATCH 0146/1172] Fix double INCREF-ing when unpacking. --- msgpack.c | 20 +++++++++++--------- msgpack.pyx | 4 ++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/msgpack.c b/msgpack.c index 50ec6fc..e044f18 100644 --- a/msgpack.c +++ b/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Mon Jun 8 01:28:30 2009 */ +/* Generated by Cython 0.11.2 on Mon Jun 8 12:41:02 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -1724,7 +1724,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx PyObject *__pyx_r = NULL; const char* __pyx_t_1; Py_ssize_t __pyx_t_2; - PyObject *__pyx_t_3; + PyObject *__pyx_t_3 = NULL; __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; @@ -1752,7 +1752,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) + * return template_data(&ctx) */ template_init((&__pyx_v_ctx)); @@ -1760,7 +1760,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< - * return template_data(&ctx) + * return template_data(&ctx) * */ __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 185; __pyx_clineno = __LINE__; goto __pyx_L1_error;} @@ -1769,19 +1769,21 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":186 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) # <<<<<<<<<<<<<< + * return template_data(&ctx) # <<<<<<<<<<<<<< * * def unpack(object stream): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = template_data((&__pyx_v_ctx)); - __Pyx_INCREF(((PyObject *)__pyx_t_3)); - __pyx_r = ((PyObject *)__pyx_t_3); + __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 186; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); __Pyx_AddTraceback("msgpack.unpacks"); __pyx_r = NULL; __pyx_L0:; @@ -1791,7 +1793,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx } /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":188 - * return template_data(&ctx) + * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< * """unpack from stream.""" diff --git a/msgpack.pyx b/msgpack.pyx index c454c5d..8b2006a 100644 --- a/msgpack.pyx +++ b/msgpack.pyx @@ -173,7 +173,7 @@ cdef extern from "unpack.h": int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) void template_init(template_context* ctx) - PyObject* template_data(template_context* ctx) + object template_data(template_context* ctx) def unpacks(object packed_bytes): @@ -183,7 +183,7 @@ def unpacks(object packed_bytes): cdef size_t off = 0 template_init(&ctx) template_execute(&ctx, p, len(packed_bytes), &off) - return template_data(&ctx) + return template_data(&ctx) def unpack(object stream): """unpack from stream.""" From 9c9393bff919fa4ad9e8ff64feaf28859a0c22e6 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 13:21:38 +0900 Subject: [PATCH 0147/1172] Fix setup script doesn't work. --- python/MANIFEST | 8 ++++++++ python/setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 python/MANIFEST diff --git a/python/MANIFEST b/python/MANIFEST new file mode 100644 index 0000000..c7dd3a3 --- /dev/null +++ b/python/MANIFEST @@ -0,0 +1,8 @@ +msgpack.c +setup.py +pack.h +unpack.h +msgpack/pack_define.h +msgpack/pack_template.h +msgpack/unpack_define.h +msgpack/unpack_template.h diff --git a/python/setup.py b/python/setup.py index 8bfdf82..e5651a0 100644 --- a/python/setup.py +++ b/python/setup.py @@ -24,6 +24,6 @@ setup(name='msgpack', author_email='songofacandy@gmail.com', version=version, ext_modules=[msgpack_mod], - description='The MessagePack serializer/desirializer.', + description=desc, long_description=long_desc, ) From a06a5ce46f1b87a8684c347657a3a748da84bf58 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 13:21:38 +0900 Subject: [PATCH 0148/1172] Fix setup script doesn't work. --- MANIFEST | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 MANIFEST diff --git a/MANIFEST b/MANIFEST new file mode 100644 index 0000000..c7dd3a3 --- /dev/null +++ b/MANIFEST @@ -0,0 +1,8 @@ +msgpack.c +setup.py +pack.h +unpack.h +msgpack/pack_define.h +msgpack/pack_template.h +msgpack/unpack_define.h +msgpack/unpack_template.h diff --git a/setup.py b/setup.py index 8bfdf82..e5651a0 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,6 @@ setup(name='msgpack', author_email='songofacandy@gmail.com', version=version, ext_modules=[msgpack_mod], - description='The MessagePack serializer/desirializer.', + description=desc, long_description=long_desc, ) From 85ca59411868ab84d874ad28e56213f5374e5ca8 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 9 Jun 2009 13:12:29 +0900 Subject: [PATCH 0149/1172] free buffer when packer deleted. --- python/msgpack.c | 653 +++++++++++++++++++++++++++++---------------- python/msgpack.pyx | 9 +- 2 files changed, 437 insertions(+), 225 deletions(-) diff --git a/python/msgpack.c b/python/msgpack.c index e044f18..821f65b 100644 --- a/python/msgpack.c +++ b/python/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Mon Jun 8 12:41:02 2009 */ +/* Generated by Cython 0.11.2 on Tue Jun 9 13:10:11 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -317,13 +317,15 @@ static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); static void __Pyx_WriteUnraisable(const char *name); /*proto*/ +static int __Pyx_SetVtable(PyObject *dict, void *vtable); /*proto*/ + static void __Pyx_AddTraceback(const char *funcname); /*proto*/ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ /* Type declarations */ -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -333,6 +335,7 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ struct __pyx_obj_7msgpack_Packer { PyObject_HEAD + struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtab; char *buff; unsigned int length; unsigned int allocated; @@ -340,7 +343,7 @@ struct __pyx_obj_7msgpack_Packer { PyObject *strm; }; -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":193 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 * return unpacks(packed) * * cdef class Unpacker: # <<<<<<<<<<<<<< @@ -351,11 +354,26 @@ struct __pyx_obj_7msgpack_Packer { struct __pyx_obj_7msgpack_Unpacker { PyObject_HEAD }; + + +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 + * cdef int BUFF_SIZE=2*1024 + * + * cdef class Packer: # <<<<<<<<<<<<<< + * """Packer that pack data into strm. + * + */ + +struct __pyx_vtabstruct_7msgpack_Packer { + PyObject *(*__pack)(struct __pyx_obj_7msgpack_Packer *, PyObject *); +}; +static struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtabptr_7msgpack_Packer; /* Module declarations from msgpack */ static PyTypeObject *__pyx_ptype_7msgpack_Packer = 0; static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; static int __pyx_v_7msgpack_BUFF_SIZE; +static PyObject *__pyx_k_1 = 0; static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ #define __Pyx_MODULE_NAME "msgpack" int __pyx_module_is_main_msgpack = 0; @@ -365,6 +383,8 @@ static char __pyx_k___main__[] = "__main__"; static PyObject *__pyx_kp___main__; static char __pyx_k___init__[] = "__init__"; static PyObject *__pyx_kp___init__; +static char __pyx_k___del__[] = "__del__"; +static PyObject *__pyx_kp___del__; static char __pyx_k_flush[] = "flush"; static PyObject *__pyx_kp_flush; static char __pyx_k_pack_list[] = "pack_list"; @@ -381,6 +401,8 @@ static char __pyx_k_size[] = "size"; static PyObject *__pyx_kp_size; static char __pyx_k_len[] = "len"; static PyObject *__pyx_kp_len; +static char __pyx_k_obj[] = "obj"; +static PyObject *__pyx_kp_obj; static char __pyx_k_o[] = "o"; static PyObject *__pyx_kp_o; static char __pyx_k_stream[] = "stream"; @@ -397,8 +419,8 @@ static char __pyx_k_unpacks[] = "unpacks"; static PyObject *__pyx_kp_unpacks; static char __pyx_k_write[] = "write"; static PyObject *__pyx_kp_write; -static char __pyx_k_1[] = "flush"; -static PyObject *__pyx_kp_1; +static char __pyx_k_2[] = "flush"; +static PyObject *__pyx_kp_2; static char __pyx_k_encode[] = "encode"; static PyObject *__pyx_kp_encode; static char __pyx_k_iteritems[] = "iteritems"; @@ -411,12 +433,12 @@ static char __pyx_k_read[] = "read"; static PyObject *__pyx_kp_read; static PyObject *__pyx_builtin_staticmethod; static PyObject *__pyx_builtin_TypeError; -static PyObject *__pyx_kp_2; static PyObject *__pyx_kp_3; -static char __pyx_k_2[] = "utf-8"; -static char __pyx_k_3[] = "can't serialize %r"; +static PyObject *__pyx_kp_4; +static char __pyx_k_3[] = "utf-8"; +static char __pyx_k_4[] = "can't serialize %r"; -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":51 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 * cdef object strm * * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< @@ -478,7 +500,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return -1; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":52 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 * * def __init__(self, strm, int size=0): * if size <= 0: # <<<<<<<<<<<<<< @@ -488,7 +510,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __pyx_t_1 = (__pyx_v_size <= 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":53 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 * def __init__(self, strm, int size=0): * if size <= 0: * size = BUFF_SIZE # <<<<<<<<<<<<<< @@ -500,7 +522,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * } __pyx_L6:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":55 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 * size = BUFF_SIZE * * self.strm = strm # <<<<<<<<<<<<<< @@ -513,7 +535,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":56 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 * * self.strm = strm * self.buff = malloc(size) # <<<<<<<<<<<<<< @@ -522,7 +544,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":57 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 * self.strm = strm * self.buff = malloc(size) * self.allocated = size # <<<<<<<<<<<<<< @@ -531,7 +553,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":58 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 * self.buff = malloc(size) * self.allocated = size * self.length = 0 # <<<<<<<<<<<<<< @@ -540,12 +562,12 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":60 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 * self.length = 0 * * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< * - * + * def __del__(self): */ msgpack_packer_init((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), ((void *)__pyx_v_self), ((int (*)(void *, const char*, unsigned int))__pyx_f_7msgpack__packer_write)); @@ -554,8 +576,36 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":63 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 + * msgpack_packer_init(&self.pk, self, _packer_write) * + * def __del__(self): # <<<<<<<<<<<<<< + * free(self.buff); + * + */ + +static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ +static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused) { + PyObject *__pyx_r = NULL; + __Pyx_SetupRefcountContext("__del__"); + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 + * + * def __del__(self): + * free(self.buff); # <<<<<<<<<<<<<< + * + * def flush(self): + */ + free(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff); + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 + * free(self.buff); * * def flush(self): # <<<<<<<<<<<<<< * """Flash local buffer and output stream if it has 'flush()' method.""" @@ -572,7 +622,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("flush"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":65 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 * def flush(self): * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: # <<<<<<<<<<<<<< @@ -582,29 +632,29 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":66 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< * self.length = 0 * if hasattr(self.strm, 'flush'): */ - __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_4)); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":67 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 # <<<<<<<<<<<<<< @@ -616,26 +666,26 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec } __pyx_L5:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":68 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< * self.strm.flush() * */ - __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_1); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":69 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 * self.length = 0 * if hasattr(self.strm, 'flush'): * self.strm.flush() # <<<<<<<<<<<<<< * * def pack_list(self, len): */ - __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -657,7 +707,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":71 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 * self.strm.flush() * * def pack_list(self, len): # <<<<<<<<<<<<<< @@ -672,14 +722,14 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_list"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":84 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 * packer.pack(['foo', 'bar']) * """ * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< * * def pack_dict(self, len): */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); __pyx_r = Py_None; __Pyx_INCREF(Py_None); @@ -693,7 +743,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":86 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 * msgpack_pack_array(&self.pk, len) * * def pack_dict(self, len): # <<<<<<<<<<<<<< @@ -708,14 +758,14 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_dict"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":99 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 * packer.pack({'foo', 'bar'}) * """ * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< * - * def pack(self, object o): + * cdef __pack(self, object o): */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); __pyx_r = Py_None; __Pyx_INCREF(Py_None); @@ -729,16 +779,15 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":101 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 * msgpack_pack_map(&self.pk, len) * - * def pack(self, object o): # <<<<<<<<<<<<<< + * cdef __pack(self, object o): # <<<<<<<<<<<<<< * cdef long long intval * cdef double fval */ -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o); /*proto*/ -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o) { +static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Packer *__pyx_v_self, PyObject *__pyx_v_o) { PY_LONG_LONG __pyx_v_intval; double __pyx_v_fval; char *__pyx_v_rawval; @@ -758,12 +807,12 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject PyObject *__pyx_t_8 = NULL; int __pyx_t_9; int __pyx_t_10; - __Pyx_SetupRefcountContext("pack"); + __Pyx_SetupRefcountContext("__pack"); __Pyx_INCREF(__pyx_v_o); __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":106 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 * cdef char* rawval * * if o is None: # <<<<<<<<<<<<<< @@ -773,66 +822,66 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = (__pyx_v_o == Py_None); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":107 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 * * if o is None: * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< * elif o is True: * msgpack_pack_true(&self.pk) */ - msgpack_pack_nil((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); - goto __pyx_L5; + msgpack_pack_nil((&__pyx_v_self->pk)); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":108 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 * if o is None: * msgpack_pack_nil(&self.pk) * elif o is True: # <<<<<<<<<<<<<< * msgpack_pack_true(&self.pk) * elif o is False: */ - __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_t_1 = (__pyx_v_o == __pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":109 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 * msgpack_pack_nil(&self.pk) * elif o is True: * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< * elif o is False: * msgpack_pack_false(&self.pk) */ - msgpack_pack_true((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); - goto __pyx_L5; + msgpack_pack_true((&__pyx_v_self->pk)); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":110 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 * elif o is True: * msgpack_pack_true(&self.pk) * elif o is False: # <<<<<<<<<<<<<< * msgpack_pack_false(&self.pk) * elif isinstance(o, long): */ - __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_t_1 = (__pyx_v_o == __pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":111 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 * msgpack_pack_true(&self.pk) * elif o is False: * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< * elif isinstance(o, long): * intval = o */ - msgpack_pack_false((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); - goto __pyx_L5; + msgpack_pack_false((&__pyx_v_self->pk)); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":112 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 * elif o is False: * msgpack_pack_false(&self.pk) * elif isinstance(o, long): # <<<<<<<<<<<<<< @@ -842,28 +891,28 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":113 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 * msgpack_pack_false(&self.pk) * elif isinstance(o, long): * intval = o # <<<<<<<<<<<<<< * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":114 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 * elif isinstance(o, long): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< * elif isinstance(o, int): * intval = o */ - msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); - goto __pyx_L5; + msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":115 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): # <<<<<<<<<<<<<< @@ -873,28 +922,28 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":116 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): * intval = o # <<<<<<<<<<<<<< * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":117 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 * elif isinstance(o, int): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< * elif isinstance(o, float): * fval = 9 */ - msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); - goto __pyx_L5; + msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":118 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< @@ -904,7 +953,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":119 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): * fval = 9 # <<<<<<<<<<<<<< @@ -913,18 +962,18 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject */ __pyx_v_fval = 9; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":120 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 * elif isinstance(o, float): * fval = 9 * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< * elif isinstance(o, str): * rawval = o */ - msgpack_pack_double((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_fval); - goto __pyx_L5; + msgpack_pack_double((&__pyx_v_self->pk), __pyx_v_fval); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":121 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 * fval = 9 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< @@ -934,39 +983,39 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":122 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): * rawval = o # <<<<<<<<<<<<<< * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_4; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":123 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 * elif isinstance(o, str): * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":124 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< * elif isinstance(o, unicode): * o = o.encode('utf-8') */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L5; + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":125 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): # <<<<<<<<<<<<<< @@ -976,21 +1025,21 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":126 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): * o = o.encode('utf-8') # <<<<<<<<<<<<<< * rawval = o * msgpack_pack_raw(&self.pk, len(o)) */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_6)); - __Pyx_INCREF(__pyx_kp_2); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_2); - __Pyx_GIVEREF(__pyx_kp_2); - __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_INCREF(__pyx_kp_3); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_3); + __Pyx_GIVEREF(__pyx_kp_3); + __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; @@ -998,39 +1047,39 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_v_o = __pyx_t_7; __pyx_t_7 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":127 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 * elif isinstance(o, unicode): * o = o.encode('utf-8') * rawval = o # <<<<<<<<<<<<<< * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_4; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":128 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 * o = o.encode('utf-8') * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":129 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L5; + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":130 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): # <<<<<<<<<<<<<< @@ -1040,32 +1089,32 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":131 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< * for k,v in o.iteritems(): * self.pack(k) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":132 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): # <<<<<<<<<<<<<< * self.pack(k) * self.pack(v) */ - __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); - __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; @@ -1079,7 +1128,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject } else { __pyx_t_6 = PyIter_Next(__pyx_t_7); if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } __Pyx_GOTREF(__pyx_t_6); @@ -1096,14 +1145,14 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_v_v = __pyx_3; __pyx_3 = 0; } else { - __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_2); - __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_3); - if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_1); __pyx_1 = 0; __Pyx_DECREF(__pyx_v_k); __pyx_v_k = __pyx_2; @@ -1113,51 +1162,51 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_3 = 0; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":133 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): * self.pack(k) # <<<<<<<<<<<<<< * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): */ - __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_k); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); __Pyx_GIVEREF(__pyx_v_k); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":134 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 * for k,v in o.iteritems(): * self.pack(k) * self.pack(v) # <<<<<<<<<<<<<< * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) */ - __pyx_t_8 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L5; + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":135 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 * self.pack(k) * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< @@ -1173,17 +1222,17 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject } if (__pyx_t_10) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":136 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< * for v in o: * self.pack(v) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":137 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) * for v in o: # <<<<<<<<<<<<<< @@ -1193,7 +1242,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); } for (;;) { @@ -1206,7 +1255,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject } else { __pyx_t_6 = PyIter_Next(__pyx_t_7); if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } __Pyx_GOTREF(__pyx_t_6); @@ -1215,51 +1264,51 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_v_v = __pyx_t_6; __pyx_t_6 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":138 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) * for v in o: * self.pack(v) # <<<<<<<<<<<<<< * else: * # TODO: Serialize with defalt() like simplejson. */ - __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L5; + goto __pyx_L3; } /*else*/ { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":141 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 * else: * # TODO: Serialize with defalt() like simplejson. * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + * def pack(self, obj, flush=True): */ - __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_7)); __Pyx_INCREF(__pyx_v_o); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_8 = PyNumber_Remainder(__pyx_kp_3, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } - __pyx_L5:; + __pyx_L3:; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; @@ -1271,8 +1320,8 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __Pyx_XDECREF(__pyx_t_6); __Pyx_XDECREF(__pyx_t_7); __Pyx_XDECREF(__pyx_t_8); - __Pyx_AddTraceback("msgpack.Packer.pack"); - __pyx_r = NULL; + __Pyx_AddTraceback("msgpack.Packer.__pack"); + __pyx_r = 0; __pyx_L0:; __Pyx_DECREF(__pyx_v_k); __Pyx_DECREF(__pyx_v_v); @@ -1282,9 +1331,121 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":143 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * + * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< + * self.__pack(obj) + * if flush: + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_obj = 0; + PyObject *__pyx_v_flush = 0; + PyObject *__pyx_r = NULL; + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + static PyObject **__pyx_pyargnames[] = {&__pyx_kp_obj,&__pyx_kp_flush,0}; + __Pyx_SetupRefcountContext("pack"); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); + PyObject* values[2] = {0,0}; + values[1] = __pyx_k_1; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_obj); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (kw_args > 1) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_flush); + if (unlikely(value)) { values[1] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + __pyx_v_obj = values[0]; + __pyx_v_flush = values[1]; + } else { + __pyx_v_flush = __pyx_k_1; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: __pyx_v_flush = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: __pyx_v_obj = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("pack", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("msgpack.Packer.pack"); + return NULL; + __pyx_L4_argument_unpacking_done:; + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 + * + * def pack(self, obj, flush=True): + * self.__pack(obj) # <<<<<<<<<<<<<< + * if flush: + * self.flush() + */ + __pyx_t_1 = ((struct __pyx_vtabstruct_7msgpack_Packer *)((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->__pyx_vtab)->__pack(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self), __pyx_v_obj); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 + * def pack(self, obj, flush=True): + * self.__pack(obj) + * if flush: # <<<<<<<<<<<<<< + * self.flush() + * + */ + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_2) { + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 + * self.__pack(obj) + * if flush: + * self.flush() # <<<<<<<<<<<<<< + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + */ + __pyx_t_1 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_flush); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L6; + } + __pyx_L6:; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("msgpack.Packer.pack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 + * self.flush() + * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< * if packer.length + l > packer.allocated: * if packer.length > 0: @@ -1298,7 +1459,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("_packer_write"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":144 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< @@ -1308,7 +1469,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":145 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: * if packer.length > 0: # <<<<<<<<<<<<<< @@ -1318,23 +1479,23 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_packer->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":146 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 * if packer.length + l > packer.allocated: * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_4)); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; @@ -1343,7 +1504,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L4:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":147 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: # <<<<<<<<<<<<<< @@ -1353,29 +1514,29 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_l > 64); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":148 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< * packer.length = 0 * else: */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":149 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) * packer.length = 0 # <<<<<<<<<<<<<< @@ -1387,7 +1548,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":151 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 * packer.length = 0 * else: * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< @@ -1396,7 +1557,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":152 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 * else: * memcpy(packer.buff, b, l) * packer.length = l # <<<<<<<<<<<<<< @@ -1410,7 +1571,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":154 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 * packer.length = l * else: * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< @@ -1419,7 +1580,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":155 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 * else: * memcpy(packer.buff + packer.length, b, l) * packer.length += l # <<<<<<<<<<<<<< @@ -1430,7 +1591,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L3:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":156 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 * memcpy(packer.buff + packer.length, b, l) * packer.length += l * return 0 # <<<<<<<<<<<<<< @@ -1453,7 +1614,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":158 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 * return 0 * * def pack(object o, object stream): # <<<<<<<<<<<<<< @@ -1491,11 +1652,11 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar values[1] = PyDict_GetItem(__pyx_kwds, __pyx_kp_stream); if (likely(values[1])) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } __pyx_v_o = values[0]; __pyx_v_stream = values[1]; @@ -1507,62 +1668,62 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("msgpack.pack"); return NULL; __pyx_L4_argument_unpacking_done:; __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":159 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 * * def pack(object o, object stream): * packer = Packer(stream) # <<<<<<<<<<<<<< * packer.pack(o) * packer.flush() */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_stream); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_stream); __Pyx_GIVEREF(__pyx_v_stream); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_v_packer); __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":160 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 * def pack(object o, object stream): * packer = Packer(stream) * packer.pack(o) # <<<<<<<<<<<<<< * packer.flush() * */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_o); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":161 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 * packer = Packer(stream) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< * * def packs(object o): */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -1582,7 +1743,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":163 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 * packer.flush() * * def packs(object o): # <<<<<<<<<<<<<< @@ -1604,76 +1765,76 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":164 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 * * def packs(object o): * buf = StringIO() # <<<<<<<<<<<<<< * packer = Packer(buf) * packer.pack(o) */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_1); __pyx_1 = 0; __Pyx_DECREF(__pyx_v_buf); __pyx_v_buf = __pyx_t_1; __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":165 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 * def packs(object o): * buf = StringIO() * packer = Packer(buf) # <<<<<<<<<<<<<< * packer.pack(o) * packer.flush() */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_buf); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_buf); __Pyx_GIVEREF(__pyx_v_buf); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_v_packer); __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":166 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 * buf = StringIO() * packer = Packer(buf) * packer.pack(o) # <<<<<<<<<<<<<< * packer.flush() * return buf.getvalue() */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_o); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":167 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 * packer = Packer(buf) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< * return buf.getvalue() * */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":168 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 * packer.pack(o) * packer.flush() * return buf.getvalue() # <<<<<<<<<<<<<< @@ -1681,9 +1842,9 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v * cdef extern from "unpack.h": */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_3; @@ -1707,7 +1868,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":179 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 * * * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< @@ -1728,17 +1889,17 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":181 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 * def unpacks(object packed_bytes): * """Unpack packed_bytes to object. Returns unpacked object.""" * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< * cdef template_context ctx * cdef size_t off = 0 */ - __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 181; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_p = __pyx_t_1; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":183 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 * cdef const_char_ptr p = packed_bytes * cdef template_context ctx * cdef size_t off = 0 # <<<<<<<<<<<<<< @@ -1747,7 +1908,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ __pyx_v_off = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":184 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 * cdef template_context ctx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< @@ -1756,17 +1917,17 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ template_init((&__pyx_v_ctx)); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":185 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< * return template_data(&ctx) * */ - __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 185; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":186 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) * return template_data(&ctx) # <<<<<<<<<<<<<< @@ -1774,7 +1935,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx * def unpack(object stream): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 186; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __pyx_r = __pyx_t_3; __pyx_t_3 = 0; @@ -1792,7 +1953,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":188 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< @@ -1812,23 +1973,23 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_self = __pyx_self; __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":190 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 * def unpack(object stream): * """unpack from stream.""" * packed = stream.read() # <<<<<<<<<<<<<< * return unpacks(packed) * */ - __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_v_packed); __pyx_v_packed = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":191 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 * """unpack from stream.""" * packed = stream.read() * return unpacks(packed) # <<<<<<<<<<<<<< @@ -1836,14 +1997,14 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ * cdef class Unpacker: */ __Pyx_XDECREF(__pyx_r); - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_packed); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_packed); __Pyx_GIVEREF(__pyx_v_packed); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_1); __pyx_1 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; @@ -1865,12 +2026,14 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __Pyx_FinishRefcountContext(); return __pyx_r; } +static struct __pyx_vtabstruct_7msgpack_Packer __pyx_vtable_7msgpack_Packer; static PyObject *__pyx_tp_new_7msgpack_Packer(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_7msgpack_Packer *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; p = ((struct __pyx_obj_7msgpack_Packer *)o); + p->__pyx_vtab = __pyx_vtabptr_7msgpack_Packer; p->strm = Py_None; Py_INCREF(Py_None); return o; } @@ -1900,10 +2063,11 @@ static int __pyx_tp_clear_7msgpack_Packer(PyObject *o) { } static struct PyMethodDef __pyx_methods_7msgpack_Packer[] = { + {__Pyx_NAMESTR("__del__"), (PyCFunction)__pyx_pf_7msgpack_6Packer___del__, METH_NOARGS, __Pyx_DOCSTR(0)}, {__Pyx_NAMESTR("flush"), (PyCFunction)__pyx_pf_7msgpack_6Packer_flush, METH_NOARGS, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_flush)}, {__Pyx_NAMESTR("pack_list"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_list, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_list)}, {__Pyx_NAMESTR("pack_dict"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_dict, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_dict)}, - {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_O, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, {0, 0, 0, 0} }; @@ -2240,6 +2404,7 @@ static struct PyModuleDef __pyx_moduledef = { static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, + {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, {&__pyx_kp_pack_list, __pyx_k_pack_list, sizeof(__pyx_k_pack_list), 1, 1, 1}, {&__pyx_kp_pack_dict, __pyx_k_pack_dict, sizeof(__pyx_k_pack_dict), 1, 1, 1}, @@ -2248,6 +2413,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_strm, __pyx_k_strm, sizeof(__pyx_k_strm), 1, 1, 1}, {&__pyx_kp_size, __pyx_k_size, sizeof(__pyx_k_size), 1, 1, 1}, {&__pyx_kp_len, __pyx_k_len, sizeof(__pyx_k_len), 1, 1, 1}, + {&__pyx_kp_obj, __pyx_k_obj, sizeof(__pyx_k_obj), 1, 1, 1}, {&__pyx_kp_o, __pyx_k_o, sizeof(__pyx_k_o), 1, 1, 1}, {&__pyx_kp_stream, __pyx_k_stream, sizeof(__pyx_k_stream), 1, 1, 1}, {&__pyx_kp_packed_bytes, __pyx_k_packed_bytes, sizeof(__pyx_k_packed_bytes), 1, 1, 1}, @@ -2256,19 +2422,19 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 1, 1, 1}, {&__pyx_kp_unpacks, __pyx_k_unpacks, sizeof(__pyx_k_unpacks), 1, 1, 1}, {&__pyx_kp_write, __pyx_k_write, sizeof(__pyx_k_write), 1, 1, 1}, - {&__pyx_kp_1, __pyx_k_1, sizeof(__pyx_k_1), 0, 1, 0}, + {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 1, 0}, {&__pyx_kp_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 1, 1, 1}, {&__pyx_kp_iteritems, __pyx_k_iteritems, sizeof(__pyx_k_iteritems), 1, 1, 1}, {&__pyx_kp_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 1, 1, 1}, {&__pyx_kp_getvalue, __pyx_k_getvalue, sizeof(__pyx_k_getvalue), 1, 1, 1}, {&__pyx_kp_read, __pyx_k_read, sizeof(__pyx_k_read), 1, 1, 1}, - {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 0, 0}, {&__pyx_kp_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 0}, + {&__pyx_kp_4, __pyx_k_4, sizeof(__pyx_k_4), 0, 0, 0}, {0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} return 0; __pyx_L1_error:; return -1; @@ -2337,17 +2503,24 @@ PyMODINIT_FUNC PyInit_msgpack(void) /*--- Global init code ---*/ /*--- Function export code ---*/ /*--- Type init code ---*/ + __pyx_vtabptr_7msgpack_Packer = &__pyx_vtable_7msgpack_Packer; + #if PY_MAJOR_VERSION >= 3 + __pyx_vtable_7msgpack_Packer.__pack = (PyObject *(*)(struct __pyx_obj_7msgpack_Packer *, PyObject *))__pyx_f_7msgpack_6Packer___pack; + #else + *(void(**)(void))&__pyx_vtable_7msgpack_Packer.__pack = (void(*)(void))__pyx_f_7msgpack_6Packer___pack; + #endif if (PyType_Ready(&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetVtable(__pyx_type_7msgpack_Packer.tp_dict, __pyx_vtabptr_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__Pyx_SetAttrString(__pyx_m, "Packer", (PyObject *)&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_7msgpack_Packer = &__pyx_type_7msgpack_Packer; - if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_7msgpack_Unpacker = &__pyx_type_7msgpack_Unpacker; /*--- Type import code ---*/ /*--- Function import code ---*/ /*--- Execution code ---*/ - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":3 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 * # coding: utf-8 * * from cStringIO import StringIO # <<<<<<<<<<<<<< @@ -2368,7 +2541,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __Pyx_DECREF(__pyx_2); __pyx_2 = 0; __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":37 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 * * * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< @@ -2377,22 +2550,35 @@ PyMODINIT_FUNC PyInit_msgpack(void) */ __pyx_v_7msgpack_BUFF_SIZE = 2048; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":195 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 + * raise TypeError, "can't serialize %r" % (o,) + * + * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< + * self.__pack(obj) + * if flush: + */ + __pyx_t_1 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_k_1 = __pyx_t_1; + __pyx_t_1 = 0; + __Pyx_GIVEREF(__pyx_k_1); + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 * cdef class Unpacker: * """Do nothing. This function is for symmetric to Packer""" * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_1); __Pyx_GIVEREF(__pyx_1); __pyx_1 = 0; - __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; PyType_Modified(__pyx_ptype_7msgpack_Unpacker); goto __pyx_L0; @@ -3037,6 +3223,25 @@ static void __Pyx_WriteUnraisable(const char *name) { } } +static int __Pyx_SetVtable(PyObject *dict, void *vtable) { + PyObject *pycobj = 0; + int result; + + pycobj = PyCObject_FromVoidPtr(vtable, 0); + if (!pycobj) + goto bad; + if (PyDict_SetItemString(dict, "__pyx_vtable__", pycobj) < 0) + goto bad; + result = 0; + goto done; + +bad: + result = -1; +done: + Py_XDECREF(pycobj); + return result; +} + #include "compile.h" #include "frameobject.h" #include "traceback.h" diff --git a/python/msgpack.pyx b/python/msgpack.pyx index 8b2006a..f24f403 100644 --- a/python/msgpack.pyx +++ b/python/msgpack.pyx @@ -59,6 +59,8 @@ cdef class Packer: msgpack_packer_init(&self.pk, self, _packer_write) + def __del__(self): + free(self.buff); def flush(self): """Flash local buffer and output stream if it has 'flush()' method.""" @@ -98,7 +100,7 @@ cdef class Packer: """ msgpack_pack_map(&self.pk, len) - def pack(self, object o): + cdef __pack(self, object o): cdef long long intval cdef double fval cdef char* rawval @@ -140,6 +142,11 @@ cdef class Packer: # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) + def pack(self, obj, flush=True): + self.__pack(obj) + if flush: + self.flush() + cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): if packer.length + l > packer.allocated: if packer.length > 0: From 85b68fbf8c33fb7daaaa1caafe50be8361c4b1d1 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 9 Jun 2009 13:12:29 +0900 Subject: [PATCH 0150/1172] free buffer when packer deleted. --- msgpack.c | 653 ++++++++++++++++++++++++++++++++++------------------ msgpack.pyx | 9 +- 2 files changed, 437 insertions(+), 225 deletions(-) diff --git a/msgpack.c b/msgpack.c index e044f18..821f65b 100644 --- a/msgpack.c +++ b/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Mon Jun 8 12:41:02 2009 */ +/* Generated by Cython 0.11.2 on Tue Jun 9 13:10:11 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -317,13 +317,15 @@ static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); static void __Pyx_WriteUnraisable(const char *name); /*proto*/ +static int __Pyx_SetVtable(PyObject *dict, void *vtable); /*proto*/ + static void __Pyx_AddTraceback(const char *funcname); /*proto*/ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ /* Type declarations */ -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -333,6 +335,7 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ struct __pyx_obj_7msgpack_Packer { PyObject_HEAD + struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtab; char *buff; unsigned int length; unsigned int allocated; @@ -340,7 +343,7 @@ struct __pyx_obj_7msgpack_Packer { PyObject *strm; }; -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":193 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 * return unpacks(packed) * * cdef class Unpacker: # <<<<<<<<<<<<<< @@ -351,11 +354,26 @@ struct __pyx_obj_7msgpack_Packer { struct __pyx_obj_7msgpack_Unpacker { PyObject_HEAD }; + + +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 + * cdef int BUFF_SIZE=2*1024 + * + * cdef class Packer: # <<<<<<<<<<<<<< + * """Packer that pack data into strm. + * + */ + +struct __pyx_vtabstruct_7msgpack_Packer { + PyObject *(*__pack)(struct __pyx_obj_7msgpack_Packer *, PyObject *); +}; +static struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtabptr_7msgpack_Packer; /* Module declarations from msgpack */ static PyTypeObject *__pyx_ptype_7msgpack_Packer = 0; static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; static int __pyx_v_7msgpack_BUFF_SIZE; +static PyObject *__pyx_k_1 = 0; static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ #define __Pyx_MODULE_NAME "msgpack" int __pyx_module_is_main_msgpack = 0; @@ -365,6 +383,8 @@ static char __pyx_k___main__[] = "__main__"; static PyObject *__pyx_kp___main__; static char __pyx_k___init__[] = "__init__"; static PyObject *__pyx_kp___init__; +static char __pyx_k___del__[] = "__del__"; +static PyObject *__pyx_kp___del__; static char __pyx_k_flush[] = "flush"; static PyObject *__pyx_kp_flush; static char __pyx_k_pack_list[] = "pack_list"; @@ -381,6 +401,8 @@ static char __pyx_k_size[] = "size"; static PyObject *__pyx_kp_size; static char __pyx_k_len[] = "len"; static PyObject *__pyx_kp_len; +static char __pyx_k_obj[] = "obj"; +static PyObject *__pyx_kp_obj; static char __pyx_k_o[] = "o"; static PyObject *__pyx_kp_o; static char __pyx_k_stream[] = "stream"; @@ -397,8 +419,8 @@ static char __pyx_k_unpacks[] = "unpacks"; static PyObject *__pyx_kp_unpacks; static char __pyx_k_write[] = "write"; static PyObject *__pyx_kp_write; -static char __pyx_k_1[] = "flush"; -static PyObject *__pyx_kp_1; +static char __pyx_k_2[] = "flush"; +static PyObject *__pyx_kp_2; static char __pyx_k_encode[] = "encode"; static PyObject *__pyx_kp_encode; static char __pyx_k_iteritems[] = "iteritems"; @@ -411,12 +433,12 @@ static char __pyx_k_read[] = "read"; static PyObject *__pyx_kp_read; static PyObject *__pyx_builtin_staticmethod; static PyObject *__pyx_builtin_TypeError; -static PyObject *__pyx_kp_2; static PyObject *__pyx_kp_3; -static char __pyx_k_2[] = "utf-8"; -static char __pyx_k_3[] = "can't serialize %r"; +static PyObject *__pyx_kp_4; +static char __pyx_k_3[] = "utf-8"; +static char __pyx_k_4[] = "can't serialize %r"; -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":51 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 * cdef object strm * * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< @@ -478,7 +500,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return -1; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":52 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 * * def __init__(self, strm, int size=0): * if size <= 0: # <<<<<<<<<<<<<< @@ -488,7 +510,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __pyx_t_1 = (__pyx_v_size <= 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":53 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 * def __init__(self, strm, int size=0): * if size <= 0: * size = BUFF_SIZE # <<<<<<<<<<<<<< @@ -500,7 +522,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * } __pyx_L6:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":55 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 * size = BUFF_SIZE * * self.strm = strm # <<<<<<<<<<<<<< @@ -513,7 +535,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":56 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 * * self.strm = strm * self.buff = malloc(size) # <<<<<<<<<<<<<< @@ -522,7 +544,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":57 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 * self.strm = strm * self.buff = malloc(size) * self.allocated = size # <<<<<<<<<<<<<< @@ -531,7 +553,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":58 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 * self.buff = malloc(size) * self.allocated = size * self.length = 0 # <<<<<<<<<<<<<< @@ -540,12 +562,12 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":60 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 * self.length = 0 * * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< * - * + * def __del__(self): */ msgpack_packer_init((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), ((void *)__pyx_v_self), ((int (*)(void *, const char*, unsigned int))__pyx_f_7msgpack__packer_write)); @@ -554,8 +576,36 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":63 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 + * msgpack_packer_init(&self.pk, self, _packer_write) * + * def __del__(self): # <<<<<<<<<<<<<< + * free(self.buff); + * + */ + +static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ +static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused) { + PyObject *__pyx_r = NULL; + __Pyx_SetupRefcountContext("__del__"); + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 + * + * def __del__(self): + * free(self.buff); # <<<<<<<<<<<<<< + * + * def flush(self): + */ + free(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff); + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 + * free(self.buff); * * def flush(self): # <<<<<<<<<<<<<< * """Flash local buffer and output stream if it has 'flush()' method.""" @@ -572,7 +622,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("flush"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":65 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 * def flush(self): * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: # <<<<<<<<<<<<<< @@ -582,29 +632,29 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":66 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< * self.length = 0 * if hasattr(self.strm, 'flush'): */ - __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_4)); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":67 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 # <<<<<<<<<<<<<< @@ -616,26 +666,26 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec } __pyx_L5:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":68 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< * self.strm.flush() * */ - __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_1); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":69 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 * self.length = 0 * if hasattr(self.strm, 'flush'): * self.strm.flush() # <<<<<<<<<<<<<< * * def pack_list(self, len): */ - __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -657,7 +707,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":71 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 * self.strm.flush() * * def pack_list(self, len): # <<<<<<<<<<<<<< @@ -672,14 +722,14 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_list"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":84 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 * packer.pack(['foo', 'bar']) * """ * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< * * def pack_dict(self, len): */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); __pyx_r = Py_None; __Pyx_INCREF(Py_None); @@ -693,7 +743,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":86 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 * msgpack_pack_array(&self.pk, len) * * def pack_dict(self, len): # <<<<<<<<<<<<<< @@ -708,14 +758,14 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_dict"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":99 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 * packer.pack({'foo', 'bar'}) * """ * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< * - * def pack(self, object o): + * cdef __pack(self, object o): */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); __pyx_r = Py_None; __Pyx_INCREF(Py_None); @@ -729,16 +779,15 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":101 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 * msgpack_pack_map(&self.pk, len) * - * def pack(self, object o): # <<<<<<<<<<<<<< + * cdef __pack(self, object o): # <<<<<<<<<<<<<< * cdef long long intval * cdef double fval */ -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o); /*proto*/ -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o) { +static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Packer *__pyx_v_self, PyObject *__pyx_v_o) { PY_LONG_LONG __pyx_v_intval; double __pyx_v_fval; char *__pyx_v_rawval; @@ -758,12 +807,12 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject PyObject *__pyx_t_8 = NULL; int __pyx_t_9; int __pyx_t_10; - __Pyx_SetupRefcountContext("pack"); + __Pyx_SetupRefcountContext("__pack"); __Pyx_INCREF(__pyx_v_o); __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":106 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 * cdef char* rawval * * if o is None: # <<<<<<<<<<<<<< @@ -773,66 +822,66 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = (__pyx_v_o == Py_None); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":107 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 * * if o is None: * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< * elif o is True: * msgpack_pack_true(&self.pk) */ - msgpack_pack_nil((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); - goto __pyx_L5; + msgpack_pack_nil((&__pyx_v_self->pk)); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":108 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 * if o is None: * msgpack_pack_nil(&self.pk) * elif o is True: # <<<<<<<<<<<<<< * msgpack_pack_true(&self.pk) * elif o is False: */ - __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_t_1 = (__pyx_v_o == __pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":109 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 * msgpack_pack_nil(&self.pk) * elif o is True: * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< * elif o is False: * msgpack_pack_false(&self.pk) */ - msgpack_pack_true((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); - goto __pyx_L5; + msgpack_pack_true((&__pyx_v_self->pk)); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":110 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 * elif o is True: * msgpack_pack_true(&self.pk) * elif o is False: # <<<<<<<<<<<<<< * msgpack_pack_false(&self.pk) * elif isinstance(o, long): */ - __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_t_1 = (__pyx_v_o == __pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":111 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 * msgpack_pack_true(&self.pk) * elif o is False: * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< * elif isinstance(o, long): * intval = o */ - msgpack_pack_false((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); - goto __pyx_L5; + msgpack_pack_false((&__pyx_v_self->pk)); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":112 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 * elif o is False: * msgpack_pack_false(&self.pk) * elif isinstance(o, long): # <<<<<<<<<<<<<< @@ -842,28 +891,28 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":113 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 * msgpack_pack_false(&self.pk) * elif isinstance(o, long): * intval = o # <<<<<<<<<<<<<< * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":114 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 * elif isinstance(o, long): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< * elif isinstance(o, int): * intval = o */ - msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); - goto __pyx_L5; + msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":115 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): # <<<<<<<<<<<<<< @@ -873,28 +922,28 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":116 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): * intval = o # <<<<<<<<<<<<<< * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":117 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 * elif isinstance(o, int): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< * elif isinstance(o, float): * fval = 9 */ - msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); - goto __pyx_L5; + msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":118 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< @@ -904,7 +953,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":119 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): * fval = 9 # <<<<<<<<<<<<<< @@ -913,18 +962,18 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject */ __pyx_v_fval = 9; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":120 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 * elif isinstance(o, float): * fval = 9 * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< * elif isinstance(o, str): * rawval = o */ - msgpack_pack_double((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_fval); - goto __pyx_L5; + msgpack_pack_double((&__pyx_v_self->pk), __pyx_v_fval); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":121 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 * fval = 9 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< @@ -934,39 +983,39 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":122 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): * rawval = o # <<<<<<<<<<<<<< * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_4; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":123 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 * elif isinstance(o, str): * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":124 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< * elif isinstance(o, unicode): * o = o.encode('utf-8') */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L5; + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":125 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): # <<<<<<<<<<<<<< @@ -976,21 +1025,21 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":126 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): * o = o.encode('utf-8') # <<<<<<<<<<<<<< * rawval = o * msgpack_pack_raw(&self.pk, len(o)) */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_6)); - __Pyx_INCREF(__pyx_kp_2); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_2); - __Pyx_GIVEREF(__pyx_kp_2); - __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_INCREF(__pyx_kp_3); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_3); + __Pyx_GIVEREF(__pyx_kp_3); + __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; @@ -998,39 +1047,39 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_v_o = __pyx_t_7; __pyx_t_7 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":127 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 * elif isinstance(o, unicode): * o = o.encode('utf-8') * rawval = o # <<<<<<<<<<<<<< * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_4; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":128 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 * o = o.encode('utf-8') * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":129 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L5; + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":130 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): # <<<<<<<<<<<<<< @@ -1040,32 +1089,32 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":131 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< * for k,v in o.iteritems(): * self.pack(k) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":132 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): # <<<<<<<<<<<<<< * self.pack(k) * self.pack(v) */ - __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); - __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; @@ -1079,7 +1128,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject } else { __pyx_t_6 = PyIter_Next(__pyx_t_7); if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } __Pyx_GOTREF(__pyx_t_6); @@ -1096,14 +1145,14 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_v_v = __pyx_3; __pyx_3 = 0; } else { - __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_2); - __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_3); - if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_1); __pyx_1 = 0; __Pyx_DECREF(__pyx_v_k); __pyx_v_k = __pyx_2; @@ -1113,51 +1162,51 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_3 = 0; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":133 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): * self.pack(k) # <<<<<<<<<<<<<< * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): */ - __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_k); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); __Pyx_GIVEREF(__pyx_v_k); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":134 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 * for k,v in o.iteritems(): * self.pack(k) * self.pack(v) # <<<<<<<<<<<<<< * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) */ - __pyx_t_8 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L5; + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":135 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 * self.pack(k) * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< @@ -1173,17 +1222,17 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject } if (__pyx_t_10) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":136 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< * for v in o: * self.pack(v) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":137 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) * for v in o: # <<<<<<<<<<<<<< @@ -1193,7 +1242,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); } for (;;) { @@ -1206,7 +1255,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject } else { __pyx_t_6 = PyIter_Next(__pyx_t_7); if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } __Pyx_GOTREF(__pyx_t_6); @@ -1215,51 +1264,51 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_v_v = __pyx_t_6; __pyx_t_6 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":138 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) * for v in o: * self.pack(v) # <<<<<<<<<<<<<< * else: * # TODO: Serialize with defalt() like simplejson. */ - __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L5; + goto __pyx_L3; } /*else*/ { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":141 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 * else: * # TODO: Serialize with defalt() like simplejson. * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + * def pack(self, obj, flush=True): */ - __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_7)); __Pyx_INCREF(__pyx_v_o); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_8 = PyNumber_Remainder(__pyx_kp_3, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } - __pyx_L5:; + __pyx_L3:; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; @@ -1271,8 +1320,8 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __Pyx_XDECREF(__pyx_t_6); __Pyx_XDECREF(__pyx_t_7); __Pyx_XDECREF(__pyx_t_8); - __Pyx_AddTraceback("msgpack.Packer.pack"); - __pyx_r = NULL; + __Pyx_AddTraceback("msgpack.Packer.__pack"); + __pyx_r = 0; __pyx_L0:; __Pyx_DECREF(__pyx_v_k); __Pyx_DECREF(__pyx_v_v); @@ -1282,9 +1331,121 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":143 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * + * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< + * self.__pack(obj) + * if flush: + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_obj = 0; + PyObject *__pyx_v_flush = 0; + PyObject *__pyx_r = NULL; + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + static PyObject **__pyx_pyargnames[] = {&__pyx_kp_obj,&__pyx_kp_flush,0}; + __Pyx_SetupRefcountContext("pack"); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); + PyObject* values[2] = {0,0}; + values[1] = __pyx_k_1; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_obj); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (kw_args > 1) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_flush); + if (unlikely(value)) { values[1] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + __pyx_v_obj = values[0]; + __pyx_v_flush = values[1]; + } else { + __pyx_v_flush = __pyx_k_1; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: __pyx_v_flush = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: __pyx_v_obj = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("pack", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("msgpack.Packer.pack"); + return NULL; + __pyx_L4_argument_unpacking_done:; + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 + * + * def pack(self, obj, flush=True): + * self.__pack(obj) # <<<<<<<<<<<<<< + * if flush: + * self.flush() + */ + __pyx_t_1 = ((struct __pyx_vtabstruct_7msgpack_Packer *)((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->__pyx_vtab)->__pack(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self), __pyx_v_obj); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 + * def pack(self, obj, flush=True): + * self.__pack(obj) + * if flush: # <<<<<<<<<<<<<< + * self.flush() + * + */ + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_2) { + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 + * self.__pack(obj) + * if flush: + * self.flush() # <<<<<<<<<<<<<< + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + */ + __pyx_t_1 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_flush); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L6; + } + __pyx_L6:; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("msgpack.Packer.pack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 + * self.flush() + * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< * if packer.length + l > packer.allocated: * if packer.length > 0: @@ -1298,7 +1459,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("_packer_write"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":144 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< @@ -1308,7 +1469,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":145 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: * if packer.length > 0: # <<<<<<<<<<<<<< @@ -1318,23 +1479,23 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_packer->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":146 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 * if packer.length + l > packer.allocated: * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_4)); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; @@ -1343,7 +1504,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L4:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":147 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: # <<<<<<<<<<<<<< @@ -1353,29 +1514,29 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_l > 64); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":148 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< * packer.length = 0 * else: */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":149 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) * packer.length = 0 # <<<<<<<<<<<<<< @@ -1387,7 +1548,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":151 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 * packer.length = 0 * else: * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< @@ -1396,7 +1557,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":152 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 * else: * memcpy(packer.buff, b, l) * packer.length = l # <<<<<<<<<<<<<< @@ -1410,7 +1571,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":154 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 * packer.length = l * else: * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< @@ -1419,7 +1580,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":155 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 * else: * memcpy(packer.buff + packer.length, b, l) * packer.length += l # <<<<<<<<<<<<<< @@ -1430,7 +1591,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L3:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":156 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 * memcpy(packer.buff + packer.length, b, l) * packer.length += l * return 0 # <<<<<<<<<<<<<< @@ -1453,7 +1614,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":158 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 * return 0 * * def pack(object o, object stream): # <<<<<<<<<<<<<< @@ -1491,11 +1652,11 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar values[1] = PyDict_GetItem(__pyx_kwds, __pyx_kp_stream); if (likely(values[1])) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } __pyx_v_o = values[0]; __pyx_v_stream = values[1]; @@ -1507,62 +1668,62 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("msgpack.pack"); return NULL; __pyx_L4_argument_unpacking_done:; __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":159 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 * * def pack(object o, object stream): * packer = Packer(stream) # <<<<<<<<<<<<<< * packer.pack(o) * packer.flush() */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_stream); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_stream); __Pyx_GIVEREF(__pyx_v_stream); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_v_packer); __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":160 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 * def pack(object o, object stream): * packer = Packer(stream) * packer.pack(o) # <<<<<<<<<<<<<< * packer.flush() * */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_o); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":161 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 * packer = Packer(stream) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< * * def packs(object o): */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -1582,7 +1743,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":163 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 * packer.flush() * * def packs(object o): # <<<<<<<<<<<<<< @@ -1604,76 +1765,76 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":164 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 * * def packs(object o): * buf = StringIO() # <<<<<<<<<<<<<< * packer = Packer(buf) * packer.pack(o) */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_1); __pyx_1 = 0; __Pyx_DECREF(__pyx_v_buf); __pyx_v_buf = __pyx_t_1; __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":165 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 * def packs(object o): * buf = StringIO() * packer = Packer(buf) # <<<<<<<<<<<<<< * packer.pack(o) * packer.flush() */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_buf); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_buf); __Pyx_GIVEREF(__pyx_v_buf); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_v_packer); __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":166 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 * buf = StringIO() * packer = Packer(buf) * packer.pack(o) # <<<<<<<<<<<<<< * packer.flush() * return buf.getvalue() */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_o); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":167 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 * packer = Packer(buf) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< * return buf.getvalue() * */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":168 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 * packer.pack(o) * packer.flush() * return buf.getvalue() # <<<<<<<<<<<<<< @@ -1681,9 +1842,9 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v * cdef extern from "unpack.h": */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_3; @@ -1707,7 +1868,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":179 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 * * * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< @@ -1728,17 +1889,17 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":181 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 * def unpacks(object packed_bytes): * """Unpack packed_bytes to object. Returns unpacked object.""" * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< * cdef template_context ctx * cdef size_t off = 0 */ - __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 181; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_p = __pyx_t_1; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":183 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 * cdef const_char_ptr p = packed_bytes * cdef template_context ctx * cdef size_t off = 0 # <<<<<<<<<<<<<< @@ -1747,7 +1908,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ __pyx_v_off = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":184 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 * cdef template_context ctx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< @@ -1756,17 +1917,17 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ template_init((&__pyx_v_ctx)); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":185 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< * return template_data(&ctx) * */ - __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 185; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":186 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) * return template_data(&ctx) # <<<<<<<<<<<<<< @@ -1774,7 +1935,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx * def unpack(object stream): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 186; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __pyx_r = __pyx_t_3; __pyx_t_3 = 0; @@ -1792,7 +1953,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":188 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< @@ -1812,23 +1973,23 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_self = __pyx_self; __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":190 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 * def unpack(object stream): * """unpack from stream.""" * packed = stream.read() # <<<<<<<<<<<<<< * return unpacks(packed) * */ - __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_v_packed); __pyx_v_packed = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":191 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 * """unpack from stream.""" * packed = stream.read() * return unpacks(packed) # <<<<<<<<<<<<<< @@ -1836,14 +1997,14 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ * cdef class Unpacker: */ __Pyx_XDECREF(__pyx_r); - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_packed); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_packed); __Pyx_GIVEREF(__pyx_v_packed); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_1); __pyx_1 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; @@ -1865,12 +2026,14 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __Pyx_FinishRefcountContext(); return __pyx_r; } +static struct __pyx_vtabstruct_7msgpack_Packer __pyx_vtable_7msgpack_Packer; static PyObject *__pyx_tp_new_7msgpack_Packer(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_7msgpack_Packer *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; p = ((struct __pyx_obj_7msgpack_Packer *)o); + p->__pyx_vtab = __pyx_vtabptr_7msgpack_Packer; p->strm = Py_None; Py_INCREF(Py_None); return o; } @@ -1900,10 +2063,11 @@ static int __pyx_tp_clear_7msgpack_Packer(PyObject *o) { } static struct PyMethodDef __pyx_methods_7msgpack_Packer[] = { + {__Pyx_NAMESTR("__del__"), (PyCFunction)__pyx_pf_7msgpack_6Packer___del__, METH_NOARGS, __Pyx_DOCSTR(0)}, {__Pyx_NAMESTR("flush"), (PyCFunction)__pyx_pf_7msgpack_6Packer_flush, METH_NOARGS, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_flush)}, {__Pyx_NAMESTR("pack_list"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_list, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_list)}, {__Pyx_NAMESTR("pack_dict"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_dict, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_dict)}, - {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_O, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, {0, 0, 0, 0} }; @@ -2240,6 +2404,7 @@ static struct PyModuleDef __pyx_moduledef = { static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, + {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, {&__pyx_kp_pack_list, __pyx_k_pack_list, sizeof(__pyx_k_pack_list), 1, 1, 1}, {&__pyx_kp_pack_dict, __pyx_k_pack_dict, sizeof(__pyx_k_pack_dict), 1, 1, 1}, @@ -2248,6 +2413,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_strm, __pyx_k_strm, sizeof(__pyx_k_strm), 1, 1, 1}, {&__pyx_kp_size, __pyx_k_size, sizeof(__pyx_k_size), 1, 1, 1}, {&__pyx_kp_len, __pyx_k_len, sizeof(__pyx_k_len), 1, 1, 1}, + {&__pyx_kp_obj, __pyx_k_obj, sizeof(__pyx_k_obj), 1, 1, 1}, {&__pyx_kp_o, __pyx_k_o, sizeof(__pyx_k_o), 1, 1, 1}, {&__pyx_kp_stream, __pyx_k_stream, sizeof(__pyx_k_stream), 1, 1, 1}, {&__pyx_kp_packed_bytes, __pyx_k_packed_bytes, sizeof(__pyx_k_packed_bytes), 1, 1, 1}, @@ -2256,19 +2422,19 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 1, 1, 1}, {&__pyx_kp_unpacks, __pyx_k_unpacks, sizeof(__pyx_k_unpacks), 1, 1, 1}, {&__pyx_kp_write, __pyx_k_write, sizeof(__pyx_k_write), 1, 1, 1}, - {&__pyx_kp_1, __pyx_k_1, sizeof(__pyx_k_1), 0, 1, 0}, + {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 1, 0}, {&__pyx_kp_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 1, 1, 1}, {&__pyx_kp_iteritems, __pyx_k_iteritems, sizeof(__pyx_k_iteritems), 1, 1, 1}, {&__pyx_kp_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 1, 1, 1}, {&__pyx_kp_getvalue, __pyx_k_getvalue, sizeof(__pyx_k_getvalue), 1, 1, 1}, {&__pyx_kp_read, __pyx_k_read, sizeof(__pyx_k_read), 1, 1, 1}, - {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 0, 0}, {&__pyx_kp_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 0}, + {&__pyx_kp_4, __pyx_k_4, sizeof(__pyx_k_4), 0, 0, 0}, {0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} return 0; __pyx_L1_error:; return -1; @@ -2337,17 +2503,24 @@ PyMODINIT_FUNC PyInit_msgpack(void) /*--- Global init code ---*/ /*--- Function export code ---*/ /*--- Type init code ---*/ + __pyx_vtabptr_7msgpack_Packer = &__pyx_vtable_7msgpack_Packer; + #if PY_MAJOR_VERSION >= 3 + __pyx_vtable_7msgpack_Packer.__pack = (PyObject *(*)(struct __pyx_obj_7msgpack_Packer *, PyObject *))__pyx_f_7msgpack_6Packer___pack; + #else + *(void(**)(void))&__pyx_vtable_7msgpack_Packer.__pack = (void(*)(void))__pyx_f_7msgpack_6Packer___pack; + #endif if (PyType_Ready(&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetVtable(__pyx_type_7msgpack_Packer.tp_dict, __pyx_vtabptr_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__Pyx_SetAttrString(__pyx_m, "Packer", (PyObject *)&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_7msgpack_Packer = &__pyx_type_7msgpack_Packer; - if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_7msgpack_Unpacker = &__pyx_type_7msgpack_Unpacker; /*--- Type import code ---*/ /*--- Function import code ---*/ /*--- Execution code ---*/ - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":3 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 * # coding: utf-8 * * from cStringIO import StringIO # <<<<<<<<<<<<<< @@ -2368,7 +2541,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __Pyx_DECREF(__pyx_2); __pyx_2 = 0; __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":37 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 * * * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< @@ -2377,22 +2550,35 @@ PyMODINIT_FUNC PyInit_msgpack(void) */ __pyx_v_7msgpack_BUFF_SIZE = 2048; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":195 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 + * raise TypeError, "can't serialize %r" % (o,) + * + * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< + * self.__pack(obj) + * if flush: + */ + __pyx_t_1 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_k_1 = __pyx_t_1; + __pyx_t_1 = 0; + __Pyx_GIVEREF(__pyx_k_1); + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 * cdef class Unpacker: * """Do nothing. This function is for symmetric to Packer""" * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_1); __Pyx_GIVEREF(__pyx_1); __pyx_1 = 0; - __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; PyType_Modified(__pyx_ptype_7msgpack_Unpacker); goto __pyx_L0; @@ -3037,6 +3223,25 @@ static void __Pyx_WriteUnraisable(const char *name) { } } +static int __Pyx_SetVtable(PyObject *dict, void *vtable) { + PyObject *pycobj = 0; + int result; + + pycobj = PyCObject_FromVoidPtr(vtable, 0); + if (!pycobj) + goto bad; + if (PyDict_SetItemString(dict, "__pyx_vtable__", pycobj) < 0) + goto bad; + result = 0; + goto done; + +bad: + result = -1; +done: + Py_XDECREF(pycobj); + return result; +} + #include "compile.h" #include "frameobject.h" #include "traceback.h" diff --git a/msgpack.pyx b/msgpack.pyx index 8b2006a..f24f403 100644 --- a/msgpack.pyx +++ b/msgpack.pyx @@ -59,6 +59,8 @@ cdef class Packer: msgpack_packer_init(&self.pk, self, _packer_write) + def __del__(self): + free(self.buff); def flush(self): """Flash local buffer and output stream if it has 'flush()' method.""" @@ -98,7 +100,7 @@ cdef class Packer: """ msgpack_pack_map(&self.pk, len) - def pack(self, object o): + cdef __pack(self, object o): cdef long long intval cdef double fval cdef char* rawval @@ -140,6 +142,11 @@ cdef class Packer: # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) + def pack(self, obj, flush=True): + self.__pack(obj) + if flush: + self.flush() + cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): if packer.length + l > packer.allocated: if packer.length > 0: From a1fb1507d45d27f0c08e8d39054987ed6dcaf9e7 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 10:45:07 +0900 Subject: [PATCH 0151/1172] Refactor include path. --- python/MANIFEST | 8 ++++---- python/include | 1 + python/msgpack | 1 - python/setup.py | 7 ++++++- 4 files changed, 11 insertions(+), 6 deletions(-) create mode 120000 python/include delete mode 120000 python/msgpack diff --git a/python/MANIFEST b/python/MANIFEST index c7dd3a3..dc042ae 100644 --- a/python/MANIFEST +++ b/python/MANIFEST @@ -2,7 +2,7 @@ msgpack.c setup.py pack.h unpack.h -msgpack/pack_define.h -msgpack/pack_template.h -msgpack/unpack_define.h -msgpack/unpack_template.h +include/msgpack/pack_define.h +include/msgpack/pack_template.h +include/msgpack/unpack_define.h +include/msgpack/unpack_template.h diff --git a/python/include b/python/include new file mode 120000 index 0000000..a96aa0e --- /dev/null +++ b/python/include @@ -0,0 +1 @@ +.. \ No newline at end of file diff --git a/python/msgpack b/python/msgpack deleted file mode 120000 index 430db49..0000000 --- a/python/msgpack +++ /dev/null @@ -1 +0,0 @@ -../msgpack \ No newline at end of file diff --git a/python/setup.py b/python/setup.py index e5651a0..65ca412 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,8 +1,13 @@ from distutils.core import setup, Extension +import os version = '0.0.1' -msgpack_mod = Extension('msgpack', sources=['msgpack.c']) +PACKAGE_ROOT = os.getcwdu() +INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') +msgpack_mod = Extension('msgpack', + sources=['msgpack.c'], + include_dirs=[INCLUDE_PATH]) desc = 'MessagePack serializer/desirializer.' long_desc = desc + """ From fcc645b18bcd36b6240acdfd01ce4d193f1912f3 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 10:45:07 +0900 Subject: [PATCH 0152/1172] Refactor include path. --- MANIFEST | 8 ++++---- include | 1 + msgpack | 1 - setup.py | 7 ++++++- 4 files changed, 11 insertions(+), 6 deletions(-) create mode 120000 include delete mode 120000 msgpack diff --git a/MANIFEST b/MANIFEST index c7dd3a3..dc042ae 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2,7 +2,7 @@ msgpack.c setup.py pack.h unpack.h -msgpack/pack_define.h -msgpack/pack_template.h -msgpack/unpack_define.h -msgpack/unpack_template.h +include/msgpack/pack_define.h +include/msgpack/pack_template.h +include/msgpack/unpack_define.h +include/msgpack/unpack_template.h diff --git a/include b/include new file mode 120000 index 0000000..a96aa0e --- /dev/null +++ b/include @@ -0,0 +1 @@ +.. \ No newline at end of file diff --git a/msgpack b/msgpack deleted file mode 120000 index 430db49..0000000 --- a/msgpack +++ /dev/null @@ -1 +0,0 @@ -../msgpack \ No newline at end of file diff --git a/setup.py b/setup.py index e5651a0..65ca412 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,13 @@ from distutils.core import setup, Extension +import os version = '0.0.1' -msgpack_mod = Extension('msgpack', sources=['msgpack.c']) +PACKAGE_ROOT = os.getcwdu() +INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') +msgpack_mod = Extension('msgpack', + sources=['msgpack.c'], + include_dirs=[INCLUDE_PATH]) desc = 'MessagePack serializer/desirializer.' long_desc = desc + """ From 3a9f74e79c3d1912d8c0c1ad4d1478c611caba0a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 10:58:09 +0900 Subject: [PATCH 0153/1172] Make msgpack package instead of module. --- python/MANIFEST | 4 +- python/msgpack.c | 3438 ------------------ python/msgpack/__init__.py | 3 + python/{msgpack.pyx => msgpack/_msgpack.pyx} | 0 python/{ => msgpack}/pack.h | 0 python/{ => msgpack}/unpack.h | 0 python/setup.py | 6 +- 7 files changed, 9 insertions(+), 3442 deletions(-) delete mode 100644 python/msgpack.c create mode 100644 python/msgpack/__init__.py rename python/{msgpack.pyx => msgpack/_msgpack.pyx} (100%) rename python/{ => msgpack}/pack.h (100%) rename python/{ => msgpack}/unpack.h (100%) diff --git a/python/MANIFEST b/python/MANIFEST index dc042ae..f2da7da 100644 --- a/python/MANIFEST +++ b/python/MANIFEST @@ -1,7 +1,7 @@ msgpack.c setup.py -pack.h -unpack.h +msgpack/pack.h +msgpack/unpack.h include/msgpack/pack_define.h include/msgpack/pack_template.h include/msgpack/unpack_define.h diff --git a/python/msgpack.c b/python/msgpack.c deleted file mode 100644 index 821f65b..0000000 --- a/python/msgpack.c +++ /dev/null @@ -1,3438 +0,0 @@ -/* Generated by Cython 0.11.2 on Tue Jun 9 13:10:11 2009 */ - -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#include "structmember.h" -#ifndef Py_PYTHON_H - #error Python headers needed to compile C extensions, please install development version of Python. -#endif -#ifndef PY_LONG_LONG - #define PY_LONG_LONG LONG_LONG -#endif -#ifndef DL_EXPORT - #define DL_EXPORT(t) t -#endif -#if PY_VERSION_HEX < 0x02040000 - #define METH_COEXIST 0 - #define PyDict_CheckExact(op) (Py_TYPE(op) == &PyDict_Type) -#endif -#if PY_VERSION_HEX < 0x02050000 - typedef int Py_ssize_t; - #define PY_SSIZE_T_MAX INT_MAX - #define PY_SSIZE_T_MIN INT_MIN - #define PY_FORMAT_SIZE_T "" - #define PyInt_FromSsize_t(z) PyInt_FromLong(z) - #define PyInt_AsSsize_t(o) PyInt_AsLong(o) - #define PyNumber_Index(o) PyNumber_Int(o) - #define PyIndex_Check(o) PyNumber_Check(o) -#endif -#if PY_VERSION_HEX < 0x02060000 - #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) - #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) - #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) - #define PyVarObject_HEAD_INIT(type, size) \ - PyObject_HEAD_INIT(type) size, - #define PyType_Modified(t) - - typedef struct { - void *buf; - PyObject *obj; - Py_ssize_t len; - Py_ssize_t itemsize; - int readonly; - int ndim; - char *format; - Py_ssize_t *shape; - Py_ssize_t *strides; - Py_ssize_t *suboffsets; - void *internal; - } Py_buffer; - - #define PyBUF_SIMPLE 0 - #define PyBUF_WRITABLE 0x0001 - #define PyBUF_FORMAT 0x0004 - #define PyBUF_ND 0x0008 - #define PyBUF_STRIDES (0x0010 | PyBUF_ND) - #define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES) - #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES) - #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES) - #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES) - -#endif -#if PY_MAJOR_VERSION < 3 - #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" -#else - #define __Pyx_BUILTIN_MODULE_NAME "builtins" -#endif -#if PY_MAJOR_VERSION >= 3 - #define Py_TPFLAGS_CHECKTYPES 0 - #define Py_TPFLAGS_HAVE_INDEX 0 -#endif -#if (PY_VERSION_HEX < 0x02060000) || (PY_MAJOR_VERSION >= 3) - #define Py_TPFLAGS_HAVE_NEWBUFFER 0 -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBaseString_Type PyUnicode_Type - #define PyString_Type PyBytes_Type - #define PyString_CheckExact PyBytes_CheckExact - #define PyInt_Type PyLong_Type - #define PyInt_Check(op) PyLong_Check(op) - #define PyInt_CheckExact(op) PyLong_CheckExact(op) - #define PyInt_FromString PyLong_FromString - #define PyInt_FromUnicode PyLong_FromUnicode - #define PyInt_FromLong PyLong_FromLong - #define PyInt_FromSize_t PyLong_FromSize_t - #define PyInt_FromSsize_t PyLong_FromSsize_t - #define PyInt_AsLong PyLong_AsLong - #define PyInt_AS_LONG PyLong_AS_LONG - #define PyInt_AsSsize_t PyLong_AsSsize_t - #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask - #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask - #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) -#else - #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) - #define PyBytes_Type PyString_Type -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyMethod_New(func, self, klass) PyInstanceMethod_New(func) -#endif -#if !defined(WIN32) && !defined(MS_WINDOWS) - #ifndef __stdcall - #define __stdcall - #endif - #ifndef __cdecl - #define __cdecl - #endif - #ifndef __fastcall - #define __fastcall - #endif -#else - #define _USE_MATH_DEFINES -#endif -#if PY_VERSION_HEX < 0x02050000 - #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),((char *)(n))) - #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),((char *)(n)),(a)) - #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),((char *)(n))) -#else - #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),(n)) - #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),(n),(a)) - #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),(n)) -#endif -#if PY_VERSION_HEX < 0x02050000 - #define __Pyx_NAMESTR(n) ((char *)(n)) - #define __Pyx_DOCSTR(n) ((char *)(n)) -#else - #define __Pyx_NAMESTR(n) (n) - #define __Pyx_DOCSTR(n) (n) -#endif -#ifdef __cplusplus -#define __PYX_EXTERN_C extern "C" -#else -#define __PYX_EXTERN_C extern -#endif -#include -#define __PYX_HAVE_API__msgpack -#include "stdlib.h" -#include "string.h" -#include "pack.h" -#include "unpack.h" -#define __PYX_USE_C99_COMPLEX defined(_Complex_I) - - -#ifdef __GNUC__ -#define INLINE __inline__ -#elif _WIN32 -#define INLINE __inline -#else -#define INLINE -#endif - -typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern; char is_identifier;} __Pyx_StringTabEntry; /*proto*/ - - - -static int __pyx_skip_dispatch = 0; - - -/* Type Conversion Predeclarations */ - -#if PY_MAJOR_VERSION < 3 -#define __Pyx_PyBytes_FromString PyString_FromString -#define __Pyx_PyBytes_FromStringAndSize PyString_FromStringAndSize -#define __Pyx_PyBytes_AsString PyString_AsString -#else -#define __Pyx_PyBytes_FromString PyBytes_FromString -#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize -#define __Pyx_PyBytes_AsString PyBytes_AsString -#endif - -#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False)) -static INLINE int __Pyx_PyObject_IsTrue(PyObject*); -static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x); - -#if !defined(T_PYSSIZET) -#if PY_VERSION_HEX < 0x02050000 -#define T_PYSSIZET T_INT -#elif !defined(T_LONGLONG) -#define T_PYSSIZET \ - ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ - ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : -1)) -#else -#define T_PYSSIZET \ - ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ - ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : \ - ((sizeof(Py_ssize_t) == sizeof(PY_LONG_LONG)) ? T_LONGLONG : -1))) -#endif -#endif - -#if !defined(T_SIZET) -#if !defined(T_ULONGLONG) -#define T_SIZET \ - ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ - ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : -1)) -#else -#define T_SIZET \ - ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ - ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : \ - ((sizeof(size_t) == sizeof(unsigned PY_LONG_LONG)) ? T_ULONGLONG : -1))) -#endif -#endif - -static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); -static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); -static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*); - -#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) - - -#ifdef __GNUC__ -/* Test for GCC > 2.95 */ -#if __GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)) -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) -#else /* __GNUC__ > 2 ... */ -#define likely(x) (x) -#define unlikely(x) (x) -#endif /* __GNUC__ > 2 ... */ -#else /* __GNUC__ */ -#define likely(x) (x) -#define unlikely(x) (x) -#endif /* __GNUC__ */ - -static PyObject *__pyx_m; -static PyObject *__pyx_b; -static PyObject *__pyx_empty_tuple; -static int __pyx_lineno; -static int __pyx_clineno = 0; -static const char * __pyx_cfilenm= __FILE__; -static const char *__pyx_filename; -static const char **__pyx_f; - - -#ifdef CYTHON_REFNANNY -typedef struct { - void (*INCREF)(void*, PyObject*, int); - void (*DECREF)(void*, PyObject*, int); - void (*GOTREF)(void*, PyObject*, int); - void (*GIVEREF)(void*, PyObject*, int); - void* (*NewContext)(const char*, int, const char*); - void (*FinishContext)(void**); -} __Pyx_RefnannyAPIStruct; -static __Pyx_RefnannyAPIStruct *__Pyx_Refnanny = NULL; -#define __Pyx_ImportRefcountAPI(name) (__Pyx_RefnannyAPIStruct *) PyCObject_Import((char *)name, (char *)"RefnannyAPI") -#define __Pyx_INCREF(r) __Pyx_Refnanny->INCREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_DECREF(r) __Pyx_Refnanny->DECREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_GOTREF(r) __Pyx_Refnanny->GOTREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_GIVEREF(r) __Pyx_Refnanny->GIVEREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_XDECREF(r) if((r) == NULL) ; else __Pyx_DECREF(r) -#define __Pyx_SetupRefcountContext(name) void* __pyx_refchk = __Pyx_Refnanny->NewContext((name), __LINE__, __FILE__) -#define __Pyx_FinishRefcountContext() __Pyx_Refnanny->FinishContext(&__pyx_refchk) -#else -#define __Pyx_INCREF(r) Py_INCREF(r) -#define __Pyx_DECREF(r) Py_DECREF(r) -#define __Pyx_GOTREF(r) -#define __Pyx_GIVEREF(r) -#define __Pyx_XDECREF(r) Py_XDECREF(r) -#define __Pyx_SetupRefcountContext(name) -#define __Pyx_FinishRefcountContext() -#endif /* CYTHON_REFNANNY */ -#define __Pyx_XGIVEREF(r) if((r) == NULL) ; else __Pyx_GIVEREF(r) -#define __Pyx_XGOTREF(r) if((r) == NULL) ; else __Pyx_GOTREF(r) - -static void __Pyx_RaiseDoubleKeywordsError( - const char* func_name, PyObject* kw_name); /*proto*/ - -static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, - Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); /*proto*/ - -static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, const char* function_name); /*proto*/ - -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list); /*proto*/ - -static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name); /*proto*/ - -static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); - -static INLINE void __Pyx_RaiseTooManyValuesError(void); - -static PyObject *__Pyx_UnpackItem(PyObject *, Py_ssize_t index); /*proto*/ -static int __Pyx_EndUnpack(PyObject *); /*proto*/ - -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ - -static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ -static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ - -static INLINE int __Pyx_StrEq(const char *, const char *); /*proto*/ - -static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject *); - -static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject *); - -static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject *); - -static INLINE char __Pyx_PyInt_AsChar(PyObject *); - -static INLINE short __Pyx_PyInt_AsShort(PyObject *); - -static INLINE int __Pyx_PyInt_AsInt(PyObject *); - -static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject *); - -static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject *); - -static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject *); - -static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject *); - -static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject *); - -static INLINE long __Pyx_PyInt_AsLong(PyObject *); - -static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject *); - -static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject *); - -static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); - -static void __Pyx_WriteUnraisable(const char *name); /*proto*/ - -static int __Pyx_SetVtable(PyObject *dict, void *vtable); /*proto*/ - -static void __Pyx_AddTraceback(const char *funcname); /*proto*/ - -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ - -/* Type declarations */ - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 - * cdef int BUFF_SIZE=2*1024 - * - * cdef class Packer: # <<<<<<<<<<<<<< - * """Packer that pack data into strm. - * - */ - -struct __pyx_obj_7msgpack_Packer { - PyObject_HEAD - struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtab; - char *buff; - unsigned int length; - unsigned int allocated; - struct msgpack_packer pk; - PyObject *strm; -}; - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 - * return unpacks(packed) - * - * cdef class Unpacker: # <<<<<<<<<<<<<< - * """Do nothing. This function is for symmetric to Packer""" - * unpack = staticmethod(unpacks) - */ - -struct __pyx_obj_7msgpack_Unpacker { - PyObject_HEAD -}; - - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 - * cdef int BUFF_SIZE=2*1024 - * - * cdef class Packer: # <<<<<<<<<<<<<< - * """Packer that pack data into strm. - * - */ - -struct __pyx_vtabstruct_7msgpack_Packer { - PyObject *(*__pack)(struct __pyx_obj_7msgpack_Packer *, PyObject *); -}; -static struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtabptr_7msgpack_Packer; -/* Module declarations from msgpack */ - -static PyTypeObject *__pyx_ptype_7msgpack_Packer = 0; -static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; -static int __pyx_v_7msgpack_BUFF_SIZE; -static PyObject *__pyx_k_1 = 0; -static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ -#define __Pyx_MODULE_NAME "msgpack" -int __pyx_module_is_main_msgpack = 0; - -/* Implementation of msgpack */ -static char __pyx_k___main__[] = "__main__"; -static PyObject *__pyx_kp___main__; -static char __pyx_k___init__[] = "__init__"; -static PyObject *__pyx_kp___init__; -static char __pyx_k___del__[] = "__del__"; -static PyObject *__pyx_kp___del__; -static char __pyx_k_flush[] = "flush"; -static PyObject *__pyx_kp_flush; -static char __pyx_k_pack_list[] = "pack_list"; -static PyObject *__pyx_kp_pack_list; -static char __pyx_k_pack_dict[] = "pack_dict"; -static PyObject *__pyx_kp_pack_dict; -static char __pyx_k_pack[] = "pack"; -static PyObject *__pyx_kp_pack; -static char __pyx_k_unpack[] = "unpack"; -static PyObject *__pyx_kp_unpack; -static char __pyx_k_strm[] = "strm"; -static PyObject *__pyx_kp_strm; -static char __pyx_k_size[] = "size"; -static PyObject *__pyx_kp_size; -static char __pyx_k_len[] = "len"; -static PyObject *__pyx_kp_len; -static char __pyx_k_obj[] = "obj"; -static PyObject *__pyx_kp_obj; -static char __pyx_k_o[] = "o"; -static PyObject *__pyx_kp_o; -static char __pyx_k_stream[] = "stream"; -static PyObject *__pyx_kp_stream; -static char __pyx_k_packed_bytes[] = "packed_bytes"; -static PyObject *__pyx_kp_packed_bytes; -static char __pyx_k_cStringIO[] = "cStringIO"; -static PyObject *__pyx_kp_cStringIO; -static char __pyx_k_StringIO[] = "StringIO"; -static PyObject *__pyx_kp_StringIO; -static char __pyx_k_staticmethod[] = "staticmethod"; -static PyObject *__pyx_kp_staticmethod; -static char __pyx_k_unpacks[] = "unpacks"; -static PyObject *__pyx_kp_unpacks; -static char __pyx_k_write[] = "write"; -static PyObject *__pyx_kp_write; -static char __pyx_k_2[] = "flush"; -static PyObject *__pyx_kp_2; -static char __pyx_k_encode[] = "encode"; -static PyObject *__pyx_kp_encode; -static char __pyx_k_iteritems[] = "iteritems"; -static PyObject *__pyx_kp_iteritems; -static char __pyx_k_TypeError[] = "TypeError"; -static PyObject *__pyx_kp_TypeError; -static char __pyx_k_getvalue[] = "getvalue"; -static PyObject *__pyx_kp_getvalue; -static char __pyx_k_read[] = "read"; -static PyObject *__pyx_kp_read; -static PyObject *__pyx_builtin_staticmethod; -static PyObject *__pyx_builtin_TypeError; -static PyObject *__pyx_kp_3; -static PyObject *__pyx_kp_4; -static char __pyx_k_3[] = "utf-8"; -static char __pyx_k_4[] = "can't serialize %r"; - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 - * cdef object strm - * - * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< - * if size <= 0: - * size = BUFF_SIZE - */ - -static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_strm = 0; - int __pyx_v_size; - int __pyx_r; - int __pyx_t_1; - static PyObject **__pyx_pyargnames[] = {&__pyx_kp_strm,&__pyx_kp_size,0}; - __Pyx_SetupRefcountContext("__init__"); - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); - PyObject* values[2] = {0,0}; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 0: - values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_strm); - if (likely(values[0])) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (kw_args > 1) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_size); - if (unlikely(value)) { values[1] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "__init__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - __pyx_v_strm = values[0]; - if (values[1]) { - __pyx_v_size = __Pyx_PyInt_AsInt(values[1]); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } else { - __pyx_v_size = 0; - } - } else { - __pyx_v_size = 0; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: __pyx_v_size = __Pyx_PyInt_AsInt(PyTuple_GET_ITEM(__pyx_args, 1)); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - case 1: __pyx_v_strm = PyTuple_GET_ITEM(__pyx_args, 0); - break; - default: goto __pyx_L5_argtuple_error; - } - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack.Packer.__init__"); - return -1; - __pyx_L4_argument_unpacking_done:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 - * - * def __init__(self, strm, int size=0): - * if size <= 0: # <<<<<<<<<<<<<< - * size = BUFF_SIZE - * - */ - __pyx_t_1 = (__pyx_v_size <= 0); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 - * def __init__(self, strm, int size=0): - * if size <= 0: - * size = BUFF_SIZE # <<<<<<<<<<<<<< - * - * self.strm = strm - */ - __pyx_v_size = __pyx_v_7msgpack_BUFF_SIZE; - goto __pyx_L6; - } - __pyx_L6:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 - * size = BUFF_SIZE - * - * self.strm = strm # <<<<<<<<<<<<<< - * self.buff = malloc(size) - * self.allocated = size - */ - __Pyx_INCREF(__pyx_v_strm); - __Pyx_GIVEREF(__pyx_v_strm); - __Pyx_GOTREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); - __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 - * - * self.strm = strm - * self.buff = malloc(size) # <<<<<<<<<<<<<< - * self.allocated = size - * self.length = 0 - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 - * self.strm = strm - * self.buff = malloc(size) - * self.allocated = size # <<<<<<<<<<<<<< - * self.length = 0 - * - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 - * self.buff = malloc(size) - * self.allocated = size - * self.length = 0 # <<<<<<<<<<<<<< - * - * msgpack_packer_init(&self.pk, self, _packer_write) - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 - * self.length = 0 - * - * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< - * - * def __del__(self): - */ - msgpack_packer_init((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), ((void *)__pyx_v_self), ((int (*)(void *, const char*, unsigned int))__pyx_f_7msgpack__packer_write)); - - __pyx_r = 0; - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 - * msgpack_packer_init(&self.pk, self, _packer_write) - * - * def __del__(self): # <<<<<<<<<<<<<< - * free(self.buff); - * - */ - -static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ -static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused) { - PyObject *__pyx_r = NULL; - __Pyx_SetupRefcountContext("__del__"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 - * - * def __del__(self): - * free(self.buff); # <<<<<<<<<<<<<< - * - * def flush(self): - */ - free(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff); - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 - * free(self.buff); - * - * def flush(self): # <<<<<<<<<<<<<< - * """Flash local buffer and output stream if it has 'flush()' method.""" - * if self.length > 0: - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ -static char __pyx_doc_7msgpack_6Packer_flush[] = "Flash local buffer and output stream if it has 'flush()' method."; -static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused) { - PyObject *__pyx_r = NULL; - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - __Pyx_SetupRefcountContext("flush"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 - * def flush(self): - * """Flash local buffer and output stream if it has 'flush()' method.""" - * if self.length > 0: # <<<<<<<<<<<<<< - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - * self.length = 0 - */ - __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 - * """Flash local buffer and output stream if it has 'flush()' method.""" - * if self.length > 0: - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< - * self.length = 0 - * if hasattr(self.strm, 'flush'): - */ - __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_4)); - PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_3); - __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 - * if self.length > 0: - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - * self.length = 0 # <<<<<<<<<<<<<< - * if hasattr(self.strm, 'flush'): - * self.strm.flush() - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - goto __pyx_L5; - } - __pyx_L5:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - * self.length = 0 - * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< - * self.strm.flush() - * - */ - __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 - * self.length = 0 - * if hasattr(self.strm, 'flush'): - * self.strm.flush() # <<<<<<<<<<<<<< - * - * def pack_list(self, len): - */ - __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - goto __pyx_L6; - } - __pyx_L6:; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("msgpack.Packer.flush"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 - * self.strm.flush() - * - * def pack_list(self, len): # <<<<<<<<<<<<<< - * """Start packing sequential objects. - * - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ -static char __pyx_doc_7msgpack_6Packer_pack_list[] = "Start packing sequential objects.\n\n Example:\n\n packer.pack_list(2)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack(['foo', 'bar'])\n "; -static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { - PyObject *__pyx_r = NULL; - size_t __pyx_t_1; - __Pyx_SetupRefcountContext("pack_list"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 - * packer.pack(['foo', 'bar']) - * """ - * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< - * - * def pack_dict(self, len): - */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_AddTraceback("msgpack.Packer.pack_list"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 - * msgpack_pack_array(&self.pk, len) - * - * def pack_dict(self, len): # <<<<<<<<<<<<<< - * """Start packing key-value objects. - * - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ -static char __pyx_doc_7msgpack_6Packer_pack_dict[] = "Start packing key-value objects.\n\n Example:\n\n packer.pack_dict(1)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack({'foo', 'bar'})\n "; -static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { - PyObject *__pyx_r = NULL; - size_t __pyx_t_1; - __Pyx_SetupRefcountContext("pack_dict"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 - * packer.pack({'foo', 'bar'}) - * """ - * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< - * - * cdef __pack(self, object o): - */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_AddTraceback("msgpack.Packer.pack_dict"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 - * msgpack_pack_map(&self.pk, len) - * - * cdef __pack(self, object o): # <<<<<<<<<<<<<< - * cdef long long intval - * cdef double fval - */ - -static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Packer *__pyx_v_self, PyObject *__pyx_v_o) { - PY_LONG_LONG __pyx_v_intval; - double __pyx_v_fval; - char *__pyx_v_rawval; - PyObject *__pyx_v_k; - PyObject *__pyx_v_v; - PyObject *__pyx_r = NULL; - PyObject *__pyx_1 = 0; - PyObject *__pyx_2 = 0; - PyObject *__pyx_3 = 0; - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - PY_LONG_LONG __pyx_t_3; - char *__pyx_t_4; - Py_ssize_t __pyx_t_5; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - PyObject *__pyx_t_8 = NULL; - int __pyx_t_9; - int __pyx_t_10; - __Pyx_SetupRefcountContext("__pack"); - __Pyx_INCREF(__pyx_v_o); - __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); - __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 - * cdef char* rawval - * - * if o is None: # <<<<<<<<<<<<<< - * msgpack_pack_nil(&self.pk) - * elif o is True: - */ - __pyx_t_1 = (__pyx_v_o == Py_None); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 - * - * if o is None: - * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< - * elif o is True: - * msgpack_pack_true(&self.pk) - */ - msgpack_pack_nil((&__pyx_v_self->pk)); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 - * if o is None: - * msgpack_pack_nil(&self.pk) - * elif o is True: # <<<<<<<<<<<<<< - * msgpack_pack_true(&self.pk) - * elif o is False: - */ - __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = (__pyx_v_o == __pyx_t_2); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 - * msgpack_pack_nil(&self.pk) - * elif o is True: - * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< - * elif o is False: - * msgpack_pack_false(&self.pk) - */ - msgpack_pack_true((&__pyx_v_self->pk)); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 - * elif o is True: - * msgpack_pack_true(&self.pk) - * elif o is False: # <<<<<<<<<<<<<< - * msgpack_pack_false(&self.pk) - * elif isinstance(o, long): - */ - __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = (__pyx_v_o == __pyx_t_2); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 - * msgpack_pack_true(&self.pk) - * elif o is False: - * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< - * elif isinstance(o, long): - * intval = o - */ - msgpack_pack_false((&__pyx_v_self->pk)); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 - * elif o is False: - * msgpack_pack_false(&self.pk) - * elif isinstance(o, long): # <<<<<<<<<<<<<< - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 - * msgpack_pack_false(&self.pk) - * elif isinstance(o, long): - * intval = o # <<<<<<<<<<<<<< - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, int): - */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_intval = __pyx_t_3; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 - * elif isinstance(o, long): - * intval = o - * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< - * elif isinstance(o, int): - * intval = o - */ - msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, int): # <<<<<<<<<<<<<< - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, int): - * intval = o # <<<<<<<<<<<<<< - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, float): - */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_intval = __pyx_t_3; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 - * elif isinstance(o, int): - * intval = o - * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< - * elif isinstance(o, float): - * fval = 9 - */ - msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, float): # <<<<<<<<<<<<<< - * fval = 9 - * msgpack_pack_double(&self.pk, fval) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, float): - * fval = 9 # <<<<<<<<<<<<<< - * msgpack_pack_double(&self.pk, fval) - * elif isinstance(o, str): - */ - __pyx_v_fval = 9; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 - * elif isinstance(o, float): - * fval = 9 - * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< - * elif isinstance(o, str): - * rawval = o - */ - msgpack_pack_double((&__pyx_v_self->pk), __pyx_v_fval); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 - * fval = 9 - * msgpack_pack_double(&self.pk, fval) - * elif isinstance(o, str): # <<<<<<<<<<<<<< - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 - * msgpack_pack_double(&self.pk, fval) - * elif isinstance(o, str): - * rawval = o # <<<<<<<<<<<<<< - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 - * elif isinstance(o, str): - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, unicode): - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< - * elif isinstance(o, unicode): - * o = o.encode('utf-8') - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, unicode): # <<<<<<<<<<<<<< - * o = o.encode('utf-8') - * rawval = o - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, unicode): - * o = o.encode('utf-8') # <<<<<<<<<<<<<< - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_6)); - __Pyx_INCREF(__pyx_kp_3); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_3); - __Pyx_GIVEREF(__pyx_kp_3); - __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; - __Pyx_DECREF(__pyx_v_o); - __pyx_v_o = __pyx_t_7; - __pyx_t_7 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 - * elif isinstance(o, unicode): - * o = o.encode('utf-8') - * rawval = o # <<<<<<<<<<<<<< - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 - * o = o.encode('utf-8') - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, dict): - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< - * elif isinstance(o, dict): - * msgpack_pack_map(&self.pk, len(o)) - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, dict): # <<<<<<<<<<<<<< - * msgpack_pack_map(&self.pk, len(o)) - * for k,v in o.iteritems(): - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, dict): - * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< - * for k,v in o.iteritems(): - * self.pack(k) - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 - * elif isinstance(o, dict): - * msgpack_pack_map(&self.pk, len(o)) - * for k,v in o.iteritems(): # <<<<<<<<<<<<<< - * self.pack(k) - * self.pack(v) - */ - __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); - } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - } - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - break; - } - __Pyx_GOTREF(__pyx_t_6); - } - if (PyTuple_CheckExact(__pyx_t_6) && likely(PyTuple_GET_SIZE(__pyx_t_6) == 2)) { - PyObject* tuple = __pyx_t_6; - __pyx_2 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_2); - __pyx_3 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_3); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(__pyx_v_k); - __pyx_v_k = __pyx_2; - __pyx_2 = 0; - __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_3; - __pyx_3 = 0; - } else { - __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_2); - __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_3); - if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - __Pyx_DECREF(__pyx_v_k); - __pyx_v_k = __pyx_2; - __pyx_2 = 0; - __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_3; - __pyx_3 = 0; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 - * msgpack_pack_map(&self.pk, len(o)) - * for k,v in o.iteritems(): - * self.pack(k) # <<<<<<<<<<<<<< - * self.pack(v) - * elif isinstance(o, tuple) or isinstance(o, list): - */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_k); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); - __Pyx_GIVEREF(__pyx_v_k); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 - * for k,v in o.iteritems(): - * self.pack(k) - * self.pack(v) # <<<<<<<<<<<<<< - * elif isinstance(o, tuple) or isinstance(o, list): - * msgpack_pack_array(&self.pk, len(o)) - */ - __pyx_t_8 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_v); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); - __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 - * self.pack(k) - * self.pack(v) - * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< - * msgpack_pack_array(&self.pk, len(o)) - * for v in o: - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyTuple_Type))); - if (!__pyx_t_1) { - __pyx_t_9 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); - __pyx_t_10 = __pyx_t_9; - } else { - __pyx_t_10 = __pyx_t_1; - } - if (__pyx_t_10) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 - * self.pack(v) - * elif isinstance(o, tuple) or isinstance(o, list): - * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< - * for v in o: - * self.pack(v) - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 - * elif isinstance(o, tuple) or isinstance(o, list): - * msgpack_pack_array(&self.pk, len(o)) - * for v in o: # <<<<<<<<<<<<<< - * self.pack(v) - * else: - */ - if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); - } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - } - for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - break; - } - __Pyx_GOTREF(__pyx_t_6); - } - __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_t_6; - __pyx_t_6 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 - * msgpack_pack_array(&self.pk, len(o)) - * for v in o: - * self.pack(v) # <<<<<<<<<<<<<< - * else: - * # TODO: Serialize with defalt() like simplejson. - */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_v); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); - __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L3; - } - /*else*/ { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 - * else: - * # TODO: Serialize with defalt() like simplejson. - * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< - * - * def pack(self, obj, flush=True): - */ - __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_7)); - __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); - __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_8 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; - __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - __pyx_L3:; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_2); - __Pyx_XDECREF(__pyx_3); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_XDECREF(__pyx_t_8); - __Pyx_AddTraceback("msgpack.Packer.__pack"); - __pyx_r = 0; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_k); - __Pyx_DECREF(__pyx_v_v); - __Pyx_DECREF(__pyx_v_o); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 - * raise TypeError, "can't serialize %r" % (o,) - * - * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< - * self.__pack(obj) - * if flush: - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_obj = 0; - PyObject *__pyx_v_flush = 0; - PyObject *__pyx_r = NULL; - PyObject *__pyx_t_1 = NULL; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - static PyObject **__pyx_pyargnames[] = {&__pyx_kp_obj,&__pyx_kp_flush,0}; - __Pyx_SetupRefcountContext("pack"); - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); - PyObject* values[2] = {0,0}; - values[1] = __pyx_k_1; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 0: - values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_obj); - if (likely(values[0])) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (kw_args > 1) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_flush); - if (unlikely(value)) { values[1] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - __pyx_v_obj = values[0]; - __pyx_v_flush = values[1]; - } else { - __pyx_v_flush = __pyx_k_1; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: __pyx_v_flush = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: __pyx_v_obj = PyTuple_GET_ITEM(__pyx_args, 0); - break; - default: goto __pyx_L5_argtuple_error; - } - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("pack", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack.Packer.pack"); - return NULL; - __pyx_L4_argument_unpacking_done:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 - * - * def pack(self, obj, flush=True): - * self.__pack(obj) # <<<<<<<<<<<<<< - * if flush: - * self.flush() - */ - __pyx_t_1 = ((struct __pyx_vtabstruct_7msgpack_Packer *)((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->__pyx_vtab)->__pack(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self), __pyx_v_obj); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 - * def pack(self, obj, flush=True): - * self.__pack(obj) - * if flush: # <<<<<<<<<<<<<< - * self.flush() - * - */ - __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__pyx_t_2) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 - * self.__pack(obj) - * if flush: - * self.flush() # <<<<<<<<<<<<<< - * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - */ - __pyx_t_1 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_flush); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - goto __pyx_L6; - } - __pyx_L6:; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.Packer.pack"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 - * self.flush() - * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< - * if packer.length + l > packer.allocated: - * if packer.length > 0: - */ - -static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__pyx_v_packer, const char* __pyx_v_b, unsigned int __pyx_v_l) { - int __pyx_r; - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - __Pyx_SetupRefcountContext("_packer_write"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 - * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< - * if packer.length > 0: - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - */ - __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - * if packer.length + l > packer.allocated: - * if packer.length > 0: # <<<<<<<<<<<<<< - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - * if l > 64: - */ - __pyx_t_1 = (__pyx_v_packer->length > 0); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 - * if packer.length + l > packer.allocated: - * if packer.length > 0: - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< - * if l > 64: - * packer.strm.write(PyString_FromStringAndSize(b, l)) - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_4)); - PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_3); - __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - goto __pyx_L4; - } - __pyx_L4:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 - * if packer.length > 0: - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - * if l > 64: # <<<<<<<<<<<<<< - * packer.strm.write(PyString_FromStringAndSize(b, l)) - * packer.length = 0 - */ - __pyx_t_1 = (__pyx_v_l > 64); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - * if l > 64: - * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< - * packer.length = 0 - * else: - */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_4); - __Pyx_GIVEREF(__pyx_t_4); - __pyx_t_4 = 0; - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 - * if l > 64: - * packer.strm.write(PyString_FromStringAndSize(b, l)) - * packer.length = 0 # <<<<<<<<<<<<<< - * else: - * memcpy(packer.buff, b, l) - */ - __pyx_v_packer->length = 0; - goto __pyx_L5; - } - /*else*/ { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 - * packer.length = 0 - * else: - * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< - * packer.length = l - * else: - */ - memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 - * else: - * memcpy(packer.buff, b, l) - * packer.length = l # <<<<<<<<<<<<<< - * else: - * memcpy(packer.buff + packer.length, b, l) - */ - __pyx_v_packer->length = __pyx_v_l; - } - __pyx_L5:; - goto __pyx_L3; - } - /*else*/ { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 - * packer.length = l - * else: - * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< - * packer.length += l - * return 0 - */ - memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 - * else: - * memcpy(packer.buff + packer.length, b, l) - * packer.length += l # <<<<<<<<<<<<<< - * return 0 - * - */ - __pyx_v_packer->length += __pyx_v_l; - } - __pyx_L3:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 - * memcpy(packer.buff + packer.length, b, l) - * packer.length += l - * return 0 # <<<<<<<<<<<<<< - * - * def pack(object o, object stream): - */ - __pyx_r = 0; - goto __pyx_L0; - - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_WriteUnraisable("msgpack._packer_write"); - __pyx_r = 0; - __pyx_L0:; - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 - * return 0 - * - * def pack(object o, object stream): # <<<<<<<<<<<<<< - * packer = Packer(stream) - * packer.pack(o) - */ - -static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_o = 0; - PyObject *__pyx_v_stream = 0; - PyObject *__pyx_v_packer; - PyObject *__pyx_r = NULL; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - static PyObject **__pyx_pyargnames[] = {&__pyx_kp_o,&__pyx_kp_stream,0}; - __Pyx_SetupRefcountContext("pack"); - __pyx_self = __pyx_self; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); - PyObject* values[2] = {0,0}; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 0: - values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_o); - if (likely(values[0])) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - values[1] = PyDict_GetItem(__pyx_kwds, __pyx_kp_stream); - if (likely(values[1])) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - __pyx_v_o = values[0]; - __pyx_v_stream = values[1]; - } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { - goto __pyx_L5_argtuple_error; - } else { - __pyx_v_o = PyTuple_GET_ITEM(__pyx_args, 0); - __pyx_v_stream = PyTuple_GET_ITEM(__pyx_args, 1); - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack.pack"); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 - * - * def pack(object o, object stream): - * packer = Packer(stream) # <<<<<<<<<<<<<< - * packer.pack(o) - * packer.flush() - */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_stream); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_stream); - __Pyx_GIVEREF(__pyx_v_stream); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_v_packer); - __pyx_v_packer = __pyx_t_2; - __pyx_t_2 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 - * def pack(object o, object stream): - * packer = Packer(stream) - * packer.pack(o) # <<<<<<<<<<<<<< - * packer.flush() - * - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); - __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 - * packer = Packer(stream) - * packer.pack(o) - * packer.flush() # <<<<<<<<<<<<<< - * - * def packs(object o): - */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.pack"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_packer); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 - * packer.flush() - * - * def packs(object o): # <<<<<<<<<<<<<< - * buf = StringIO() - * packer = Packer(buf) - */ - -static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o); /*proto*/ -static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o) { - PyObject *__pyx_v_buf; - PyObject *__pyx_v_packer; - PyObject *__pyx_r = NULL; - PyObject *__pyx_1 = 0; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - __Pyx_SetupRefcountContext("packs"); - __pyx_self = __pyx_self; - __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); - __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 - * - * def packs(object o): - * buf = StringIO() # <<<<<<<<<<<<<< - * packer = Packer(buf) - * packer.pack(o) - */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - __Pyx_DECREF(__pyx_v_buf); - __pyx_v_buf = __pyx_t_1; - __pyx_t_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 - * def packs(object o): - * buf = StringIO() - * packer = Packer(buf) # <<<<<<<<<<<<<< - * packer.pack(o) - * packer.flush() - */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_buf); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_buf); - __Pyx_GIVEREF(__pyx_v_buf); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_v_packer); - __pyx_v_packer = __pyx_t_2; - __pyx_t_2 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 - * buf = StringIO() - * packer = Packer(buf) - * packer.pack(o) # <<<<<<<<<<<<<< - * packer.flush() - * return buf.getvalue() - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); - __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 - * packer = Packer(buf) - * packer.pack(o) - * packer.flush() # <<<<<<<<<<<<<< - * return buf.getvalue() - * - */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 - * packer.pack(o) - * packer.flush() - * return buf.getvalue() # <<<<<<<<<<<<<< - * - * cdef extern from "unpack.h": - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_r = __pyx_t_3; - __pyx_t_3 = 0; - goto __pyx_L0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.packs"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_buf); - __Pyx_DECREF(__pyx_v_packer); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 - * - * - * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< - * """Unpack packed_bytes to object. Returns unpacked object.""" - * cdef const_char_ptr p = packed_bytes - */ - -static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes); /*proto*/ -static char __pyx_doc_7msgpack_unpacks[] = "Unpack packed_bytes to object. Returns unpacked object."; -static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes) { - const char* __pyx_v_p; - template_context __pyx_v_ctx; - size_t __pyx_v_off; - PyObject *__pyx_r = NULL; - const char* __pyx_t_1; - Py_ssize_t __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - __Pyx_SetupRefcountContext("unpacks"); - __pyx_self = __pyx_self; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 - * def unpacks(object packed_bytes): - * """Unpack packed_bytes to object. Returns unpacked object.""" - * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< - * cdef template_context ctx - * cdef size_t off = 0 - */ - __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_p = __pyx_t_1; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 - * cdef const_char_ptr p = packed_bytes - * cdef template_context ctx - * cdef size_t off = 0 # <<<<<<<<<<<<<< - * template_init(&ctx) - * template_execute(&ctx, p, len(packed_bytes), &off) - */ - __pyx_v_off = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 - * cdef template_context ctx - * cdef size_t off = 0 - * template_init(&ctx) # <<<<<<<<<<<<<< - * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) - */ - template_init((&__pyx_v_ctx)); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 - * cdef size_t off = 0 - * template_init(&ctx) - * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< - * return template_data(&ctx) - * - */ - __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 - * template_init(&ctx) - * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) # <<<<<<<<<<<<<< - * - * def unpack(object stream): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_r = __pyx_t_3; - __pyx_t_3 = 0; - goto __pyx_L0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.unpacks"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 - * return template_data(&ctx) - * - * def unpack(object stream): # <<<<<<<<<<<<<< - * """unpack from stream.""" - * packed = stream.read() - */ - -static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream); /*proto*/ -static char __pyx_doc_7msgpack_unpack[] = "unpack from stream."; -static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream) { - PyObject *__pyx_v_packed; - PyObject *__pyx_r = NULL; - PyObject *__pyx_1 = 0; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - __Pyx_SetupRefcountContext("unpack"); - __pyx_self = __pyx_self; - __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 - * def unpack(object stream): - * """unpack from stream.""" - * packed = stream.read() # <<<<<<<<<<<<<< - * return unpacks(packed) - * - */ - __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_v_packed); - __pyx_v_packed = __pyx_t_2; - __pyx_t_2 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 - * """unpack from stream.""" - * packed = stream.read() - * return unpacks(packed) # <<<<<<<<<<<<<< - * - * cdef class Unpacker: - */ - __Pyx_XDECREF(__pyx_r); - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_packed); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_packed); - __Pyx_GIVEREF(__pyx_v_packed); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("msgpack.unpack"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_packed); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} -static struct __pyx_vtabstruct_7msgpack_Packer __pyx_vtable_7msgpack_Packer; - -static PyObject *__pyx_tp_new_7msgpack_Packer(PyTypeObject *t, PyObject *a, PyObject *k) { - struct __pyx_obj_7msgpack_Packer *p; - PyObject *o = (*t->tp_alloc)(t, 0); - if (!o) return 0; - p = ((struct __pyx_obj_7msgpack_Packer *)o); - p->__pyx_vtab = __pyx_vtabptr_7msgpack_Packer; - p->strm = Py_None; Py_INCREF(Py_None); - return o; -} - -static void __pyx_tp_dealloc_7msgpack_Packer(PyObject *o) { - struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; - Py_XDECREF(p->strm); - (*Py_TYPE(o)->tp_free)(o); -} - -static int __pyx_tp_traverse_7msgpack_Packer(PyObject *o, visitproc v, void *a) { - int e; - struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; - if (p->strm) { - e = (*v)(p->strm, a); if (e) return e; - } - return 0; -} - -static int __pyx_tp_clear_7msgpack_Packer(PyObject *o) { - struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; - PyObject* tmp; - tmp = ((PyObject*)p->strm); - p->strm = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - return 0; -} - -static struct PyMethodDef __pyx_methods_7msgpack_Packer[] = { - {__Pyx_NAMESTR("__del__"), (PyCFunction)__pyx_pf_7msgpack_6Packer___del__, METH_NOARGS, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("flush"), (PyCFunction)__pyx_pf_7msgpack_6Packer_flush, METH_NOARGS, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_flush)}, - {__Pyx_NAMESTR("pack_list"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_list, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_list)}, - {__Pyx_NAMESTR("pack_dict"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_dict, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_dict)}, - {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, - {0, 0, 0, 0} -}; - -static PyNumberMethods __pyx_tp_as_number_Packer = { - 0, /*nb_add*/ - 0, /*nb_subtract*/ - 0, /*nb_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_divide*/ - #endif - 0, /*nb_remainder*/ - 0, /*nb_divmod*/ - 0, /*nb_power*/ - 0, /*nb_negative*/ - 0, /*nb_positive*/ - 0, /*nb_absolute*/ - 0, /*nb_nonzero*/ - 0, /*nb_invert*/ - 0, /*nb_lshift*/ - 0, /*nb_rshift*/ - 0, /*nb_and*/ - 0, /*nb_xor*/ - 0, /*nb_or*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_coerce*/ - #endif - 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else - 0, /*nb_long*/ - #endif - 0, /*nb_float*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_oct*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*nb_hex*/ - #endif - 0, /*nb_inplace_add*/ - 0, /*nb_inplace_subtract*/ - 0, /*nb_inplace_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_inplace_divide*/ - #endif - 0, /*nb_inplace_remainder*/ - 0, /*nb_inplace_power*/ - 0, /*nb_inplace_lshift*/ - 0, /*nb_inplace_rshift*/ - 0, /*nb_inplace_and*/ - 0, /*nb_inplace_xor*/ - 0, /*nb_inplace_or*/ - 0, /*nb_floor_divide*/ - 0, /*nb_true_divide*/ - 0, /*nb_inplace_floor_divide*/ - 0, /*nb_inplace_true_divide*/ - #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) - 0, /*nb_index*/ - #endif -}; - -static PySequenceMethods __pyx_tp_as_sequence_Packer = { - 0, /*sq_length*/ - 0, /*sq_concat*/ - 0, /*sq_repeat*/ - 0, /*sq_item*/ - 0, /*sq_slice*/ - 0, /*sq_ass_item*/ - 0, /*sq_ass_slice*/ - 0, /*sq_contains*/ - 0, /*sq_inplace_concat*/ - 0, /*sq_inplace_repeat*/ -}; - -static PyMappingMethods __pyx_tp_as_mapping_Packer = { - 0, /*mp_length*/ - 0, /*mp_subscript*/ - 0, /*mp_ass_subscript*/ -}; - -static PyBufferProcs __pyx_tp_as_buffer_Packer = { - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getreadbuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getwritebuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getsegcount*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getcharbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_getbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_releasebuffer*/ - #endif -}; - -PyTypeObject __pyx_type_7msgpack_Packer = { - PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("msgpack.Packer"), /*tp_name*/ - sizeof(struct __pyx_obj_7msgpack_Packer), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_7msgpack_Packer, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - &__pyx_tp_as_number_Packer, /*tp_as_number*/ - &__pyx_tp_as_sequence_Packer, /*tp_as_sequence*/ - &__pyx_tp_as_mapping_Packer, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - &__pyx_tp_as_buffer_Packer, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - __Pyx_DOCSTR("Packer that pack data into strm.\n\n strm must have `write(bytes)` method.\n size specifies local buffer size.\n "), /*tp_doc*/ - __pyx_tp_traverse_7msgpack_Packer, /*tp_traverse*/ - __pyx_tp_clear_7msgpack_Packer, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_7msgpack_Packer, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - __pyx_pf_7msgpack_6Packer___init__, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_7msgpack_Packer, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ -}; - -static PyObject *__pyx_tp_new_7msgpack_Unpacker(PyTypeObject *t, PyObject *a, PyObject *k) { - PyObject *o = (*t->tp_alloc)(t, 0); - if (!o) return 0; - return o; -} - -static void __pyx_tp_dealloc_7msgpack_Unpacker(PyObject *o) { - (*Py_TYPE(o)->tp_free)(o); -} - -static struct PyMethodDef __pyx_methods_7msgpack_Unpacker[] = { - {0, 0, 0, 0} -}; - -static PyNumberMethods __pyx_tp_as_number_Unpacker = { - 0, /*nb_add*/ - 0, /*nb_subtract*/ - 0, /*nb_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_divide*/ - #endif - 0, /*nb_remainder*/ - 0, /*nb_divmod*/ - 0, /*nb_power*/ - 0, /*nb_negative*/ - 0, /*nb_positive*/ - 0, /*nb_absolute*/ - 0, /*nb_nonzero*/ - 0, /*nb_invert*/ - 0, /*nb_lshift*/ - 0, /*nb_rshift*/ - 0, /*nb_and*/ - 0, /*nb_xor*/ - 0, /*nb_or*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_coerce*/ - #endif - 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else - 0, /*nb_long*/ - #endif - 0, /*nb_float*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_oct*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*nb_hex*/ - #endif - 0, /*nb_inplace_add*/ - 0, /*nb_inplace_subtract*/ - 0, /*nb_inplace_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_inplace_divide*/ - #endif - 0, /*nb_inplace_remainder*/ - 0, /*nb_inplace_power*/ - 0, /*nb_inplace_lshift*/ - 0, /*nb_inplace_rshift*/ - 0, /*nb_inplace_and*/ - 0, /*nb_inplace_xor*/ - 0, /*nb_inplace_or*/ - 0, /*nb_floor_divide*/ - 0, /*nb_true_divide*/ - 0, /*nb_inplace_floor_divide*/ - 0, /*nb_inplace_true_divide*/ - #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) - 0, /*nb_index*/ - #endif -}; - -static PySequenceMethods __pyx_tp_as_sequence_Unpacker = { - 0, /*sq_length*/ - 0, /*sq_concat*/ - 0, /*sq_repeat*/ - 0, /*sq_item*/ - 0, /*sq_slice*/ - 0, /*sq_ass_item*/ - 0, /*sq_ass_slice*/ - 0, /*sq_contains*/ - 0, /*sq_inplace_concat*/ - 0, /*sq_inplace_repeat*/ -}; - -static PyMappingMethods __pyx_tp_as_mapping_Unpacker = { - 0, /*mp_length*/ - 0, /*mp_subscript*/ - 0, /*mp_ass_subscript*/ -}; - -static PyBufferProcs __pyx_tp_as_buffer_Unpacker = { - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getreadbuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getwritebuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getsegcount*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getcharbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_getbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_releasebuffer*/ - #endif -}; - -PyTypeObject __pyx_type_7msgpack_Unpacker = { - PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("msgpack.Unpacker"), /*tp_name*/ - sizeof(struct __pyx_obj_7msgpack_Unpacker), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_7msgpack_Unpacker, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - &__pyx_tp_as_number_Unpacker, /*tp_as_number*/ - &__pyx_tp_as_sequence_Unpacker, /*tp_as_sequence*/ - &__pyx_tp_as_mapping_Unpacker, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - &__pyx_tp_as_buffer_Unpacker, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ - __Pyx_DOCSTR("Do nothing. This function is for symmetric to Packer"), /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_7msgpack_Unpacker, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_7msgpack_Unpacker, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ -}; - -static struct PyMethodDef __pyx_methods[] = { - {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("packs"), (PyCFunction)__pyx_pf_7msgpack_packs, METH_O, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("unpacks"), (PyCFunction)__pyx_pf_7msgpack_unpacks, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpacks)}, - {__Pyx_NAMESTR("unpack"), (PyCFunction)__pyx_pf_7msgpack_unpack, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpack)}, - {0, 0, 0, 0} -}; - -static void __pyx_init_filenames(void); /*proto*/ - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef __pyx_moduledef = { - PyModuleDef_HEAD_INIT, - __Pyx_NAMESTR("msgpack"), - 0, /* m_doc */ - -1, /* m_size */ - __pyx_methods /* m_methods */, - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ -}; -#endif - -static __Pyx_StringTabEntry __pyx_string_tab[] = { - {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, - {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, - {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, - {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, - {&__pyx_kp_pack_list, __pyx_k_pack_list, sizeof(__pyx_k_pack_list), 1, 1, 1}, - {&__pyx_kp_pack_dict, __pyx_k_pack_dict, sizeof(__pyx_k_pack_dict), 1, 1, 1}, - {&__pyx_kp_pack, __pyx_k_pack, sizeof(__pyx_k_pack), 1, 1, 1}, - {&__pyx_kp_unpack, __pyx_k_unpack, sizeof(__pyx_k_unpack), 1, 1, 1}, - {&__pyx_kp_strm, __pyx_k_strm, sizeof(__pyx_k_strm), 1, 1, 1}, - {&__pyx_kp_size, __pyx_k_size, sizeof(__pyx_k_size), 1, 1, 1}, - {&__pyx_kp_len, __pyx_k_len, sizeof(__pyx_k_len), 1, 1, 1}, - {&__pyx_kp_obj, __pyx_k_obj, sizeof(__pyx_k_obj), 1, 1, 1}, - {&__pyx_kp_o, __pyx_k_o, sizeof(__pyx_k_o), 1, 1, 1}, - {&__pyx_kp_stream, __pyx_k_stream, sizeof(__pyx_k_stream), 1, 1, 1}, - {&__pyx_kp_packed_bytes, __pyx_k_packed_bytes, sizeof(__pyx_k_packed_bytes), 1, 1, 1}, - {&__pyx_kp_cStringIO, __pyx_k_cStringIO, sizeof(__pyx_k_cStringIO), 1, 1, 1}, - {&__pyx_kp_StringIO, __pyx_k_StringIO, sizeof(__pyx_k_StringIO), 1, 1, 1}, - {&__pyx_kp_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 1, 1, 1}, - {&__pyx_kp_unpacks, __pyx_k_unpacks, sizeof(__pyx_k_unpacks), 1, 1, 1}, - {&__pyx_kp_write, __pyx_k_write, sizeof(__pyx_k_write), 1, 1, 1}, - {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 1, 0}, - {&__pyx_kp_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 1, 1, 1}, - {&__pyx_kp_iteritems, __pyx_k_iteritems, sizeof(__pyx_k_iteritems), 1, 1, 1}, - {&__pyx_kp_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 1, 1, 1}, - {&__pyx_kp_getvalue, __pyx_k_getvalue, sizeof(__pyx_k_getvalue), 1, 1, 1}, - {&__pyx_kp_read, __pyx_k_read, sizeof(__pyx_k_read), 1, 1, 1}, - {&__pyx_kp_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 0}, - {&__pyx_kp_4, __pyx_k_4, sizeof(__pyx_k_4), 0, 0, 0}, - {0, 0, 0, 0, 0, 0} -}; -static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - return 0; - __pyx_L1_error:; - return -1; -} - -static int __Pyx_InitGlobals(void) { - if (__Pyx_InitStrings(__pyx_string_tab) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - return 0; - __pyx_L1_error:; - return -1; -} - -#if PY_MAJOR_VERSION < 3 -PyMODINIT_FUNC initmsgpack(void); /*proto*/ -PyMODINIT_FUNC initmsgpack(void) -#else -PyMODINIT_FUNC PyInit_msgpack(void); /*proto*/ -PyMODINIT_FUNC PyInit_msgpack(void) -#endif -{ - PyObject *__pyx_1 = 0; - PyObject *__pyx_2 = 0; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - #ifdef CYTHON_REFNANNY - void* __pyx_refchk = NULL; - __Pyx_Refnanny = __Pyx_ImportRefcountAPI("refnanny"); - if (!__Pyx_Refnanny) { - PyErr_Clear(); - __Pyx_Refnanny = __Pyx_ImportRefcountAPI("Cython.Runtime.refnanny"); - if (!__Pyx_Refnanny) - Py_FatalError("failed to import refnanny module"); - } - __pyx_refchk = __Pyx_Refnanny->NewContext("PyMODINIT_FUNC PyInit_msgpack(void)", __LINE__, __FILE__); - #endif - __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /*--- Library function declarations ---*/ - __pyx_init_filenames(); - /*--- Threads initialization code ---*/ - #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS - #ifdef WITH_THREAD /* Python build with threading support? */ - PyEval_InitThreads(); - #endif - #endif - /*--- Initialize various global constants etc. ---*/ - if (unlikely(__Pyx_InitGlobals() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /*--- Module creation code ---*/ - #if PY_MAJOR_VERSION < 3 - __pyx_m = Py_InitModule4(__Pyx_NAMESTR("msgpack"), __pyx_methods, 0, 0, PYTHON_API_VERSION); - #else - __pyx_m = PyModule_Create(&__pyx_moduledef); - #endif - if (!__pyx_m) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #if PY_MAJOR_VERSION < 3 - Py_INCREF(__pyx_m); - #endif - __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); - if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - if (__pyx_module_is_main_msgpack) { - if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - } - /*--- Builtin init code ---*/ - if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_skip_dispatch = 0; - /*--- Global init code ---*/ - /*--- Function export code ---*/ - /*--- Type init code ---*/ - __pyx_vtabptr_7msgpack_Packer = &__pyx_vtable_7msgpack_Packer; - #if PY_MAJOR_VERSION >= 3 - __pyx_vtable_7msgpack_Packer.__pack = (PyObject *(*)(struct __pyx_obj_7msgpack_Packer *, PyObject *))__pyx_f_7msgpack_6Packer___pack; - #else - *(void(**)(void))&__pyx_vtable_7msgpack_Packer.__pack = (void(*)(void))__pyx_f_7msgpack_6Packer___pack; - #endif - if (PyType_Ready(&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetVtable(__pyx_type_7msgpack_Packer.tp_dict, __pyx_vtabptr_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetAttrString(__pyx_m, "Packer", (PyObject *)&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_7msgpack_Packer = &__pyx_type_7msgpack_Packer; - if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_7msgpack_Unpacker = &__pyx_type_7msgpack_Unpacker; - /*--- Type import code ---*/ - /*--- Function import code ---*/ - /*--- Execution code ---*/ - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 - * # coding: utf-8 - * - * from cStringIO import StringIO # <<<<<<<<<<<<<< - * - * cdef extern from "Python.h": - */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_kp_StringIO); - PyList_SET_ITEM(__pyx_t_1, 0, __pyx_kp_StringIO); - __Pyx_GIVEREF(__pyx_kp_StringIO); - __pyx_1 = __Pyx_Import(__pyx_kp_cStringIO, ((PyObject *)__pyx_t_1)); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __pyx_2 = PyObject_GetAttr(__pyx_1, __pyx_kp_StringIO); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_2); - if (PyObject_SetAttr(__pyx_m, __pyx_kp_StringIO, __pyx_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_2); __pyx_2 = 0; - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 - * - * - * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< - * - * cdef class Packer: - */ - __pyx_v_7msgpack_BUFF_SIZE = 2048; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 - * raise TypeError, "can't serialize %r" % (o,) - * - * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< - * self.__pack(obj) - * if flush: - */ - __pyx_t_1 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_k_1 = __pyx_t_1; - __pyx_t_1 = 0; - __Pyx_GIVEREF(__pyx_k_1); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 - * cdef class Unpacker: - * """Do nothing. This function is for symmetric to Packer""" - * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< - */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_1); - __Pyx_GIVEREF(__pyx_1); - __pyx_1 = 0; - __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - PyType_Modified(__pyx_ptype_7msgpack_Unpacker); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_2); - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("msgpack"); - Py_DECREF(__pyx_m); __pyx_m = 0; - __pyx_L0:; - __Pyx_FinishRefcountContext(); - #if PY_MAJOR_VERSION < 3 - return; - #else - return __pyx_m; - #endif -} - -static const char *__pyx_filenames[] = { - "msgpack.pyx", -}; - -/* Runtime support code */ - -static void __pyx_init_filenames(void) { - __pyx_f = __pyx_filenames; -} - -static void __Pyx_RaiseDoubleKeywordsError( - const char* func_name, - PyObject* kw_name) -{ - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION >= 3 - "%s() got multiple values for keyword argument '%U'", func_name, kw_name); - #else - "%s() got multiple values for keyword argument '%s'", func_name, - PyString_AS_STRING(kw_name)); - #endif -} - -static void __Pyx_RaiseArgtupleInvalid( - const char* func_name, - int exact, - Py_ssize_t num_min, - Py_ssize_t num_max, - Py_ssize_t num_found) -{ - Py_ssize_t num_expected; - const char *number, *more_or_less; - - if (num_found < num_min) { - num_expected = num_min; - more_or_less = "at least"; - } else { - num_expected = num_max; - more_or_less = "at most"; - } - if (exact) { - more_or_less = "exactly"; - } - number = (num_expected == 1) ? "" : "s"; - PyErr_Format(PyExc_TypeError, - #if PY_VERSION_HEX < 0x02050000 - "%s() takes %s %d positional argument%s (%d given)", - #else - "%s() takes %s %zd positional argument%s (%zd given)", - #endif - func_name, more_or_less, num_expected, number, num_found); -} - -static int __Pyx_ParseOptionalKeywords( - PyObject *kwds, - PyObject **argnames[], - PyObject *kwds2, - PyObject *values[], - Py_ssize_t num_pos_args, - const char* function_name) -{ - PyObject *key = 0, *value = 0; - Py_ssize_t pos = 0; - PyObject*** name; - PyObject*** first_kw_arg = argnames + num_pos_args; - - while (PyDict_Next(kwds, &pos, &key, &value)) { - name = first_kw_arg; - while (*name && (**name != key)) name++; - if (*name) { - values[name-argnames] = value; - } else { - #if PY_MAJOR_VERSION < 3 - if (unlikely(!PyString_CheckExact(key)) && unlikely(!PyString_Check(key))) { - #else - if (unlikely(!PyUnicode_CheckExact(key)) && unlikely(!PyUnicode_Check(key))) { - #endif - goto invalid_keyword_type; - } else { - for (name = first_kw_arg; *name; name++) { - #if PY_MAJOR_VERSION >= 3 - if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && - PyUnicode_Compare(**name, key) == 0) break; - #else - if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && - _PyString_Eq(**name, key)) break; - #endif - } - if (*name) { - values[name-argnames] = value; - } else { - /* unexpected keyword found */ - for (name=argnames; name != first_kw_arg; name++) { - if (**name == key) goto arg_passed_twice; - #if PY_MAJOR_VERSION >= 3 - if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && - PyUnicode_Compare(**name, key) == 0) goto arg_passed_twice; - #else - if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && - _PyString_Eq(**name, key)) goto arg_passed_twice; - #endif - } - if (kwds2) { - if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; - } else { - goto invalid_keyword; - } - } - } - } - } - return 0; -arg_passed_twice: - __Pyx_RaiseDoubleKeywordsError(function_name, **name); - goto bad; -invalid_keyword_type: - PyErr_Format(PyExc_TypeError, - "%s() keywords must be strings", function_name); - goto bad; -invalid_keyword: - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION < 3 - "%s() got an unexpected keyword argument '%s'", - function_name, PyString_AsString(key)); - #else - "%s() got an unexpected keyword argument '%U'", - function_name, key); - #endif -bad: - return -1; -} - -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list) { - PyObject *__import__ = 0; - PyObject *empty_list = 0; - PyObject *module = 0; - PyObject *global_dict = 0; - PyObject *empty_dict = 0; - PyObject *list; - __import__ = __Pyx_GetAttrString(__pyx_b, "__import__"); - if (!__import__) - goto bad; - if (from_list) - list = from_list; - else { - empty_list = PyList_New(0); - if (!empty_list) - goto bad; - list = empty_list; - } - global_dict = PyModule_GetDict(__pyx_m); - if (!global_dict) - goto bad; - empty_dict = PyDict_New(); - if (!empty_dict) - goto bad; - module = PyObject_CallFunctionObjArgs(__import__, - name, global_dict, empty_dict, list, NULL); -bad: - Py_XDECREF(empty_list); - Py_XDECREF(__import__); - Py_XDECREF(empty_dict); - return module; -} - -static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name) { - PyObject *result; - result = PyObject_GetAttr(dict, name); - if (!result) - PyErr_SetObject(PyExc_NameError, name); - return result; -} - -static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { - PyErr_Format(PyExc_ValueError, - #if PY_VERSION_HEX < 0x02050000 - "need more than %d value%s to unpack", (int)index, - #else - "need more than %zd value%s to unpack", index, - #endif - (index == 1) ? "" : "s"); -} - -static INLINE void __Pyx_RaiseTooManyValuesError(void) { - PyErr_SetString(PyExc_ValueError, "too many values to unpack"); -} - -static PyObject *__Pyx_UnpackItem(PyObject *iter, Py_ssize_t index) { - PyObject *item; - if (!(item = PyIter_Next(iter))) { - if (!PyErr_Occurred()) { - __Pyx_RaiseNeedMoreValuesError(index); - } - } - return item; -} - -static int __Pyx_EndUnpack(PyObject *iter) { - PyObject *item; - if ((item = PyIter_Next(iter))) { - Py_DECREF(item); - __Pyx_RaiseTooManyValuesError(); - return -1; - } - else if (!PyErr_Occurred()) - return 0; - else - return -1; -} - -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb) { - Py_XINCREF(type); - Py_XINCREF(value); - Py_XINCREF(tb); - /* First, check the traceback argument, replacing None with NULL. */ - if (tb == Py_None) { - Py_DECREF(tb); - tb = 0; - } - else if (tb != NULL && !PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto raise_error; - } - /* Next, replace a missing value with None */ - if (value == NULL) { - value = Py_None; - Py_INCREF(value); - } - #if PY_VERSION_HEX < 0x02050000 - if (!PyClass_Check(type)) - #else - if (!PyType_Check(type)) - #endif - { - /* Raising an instance. The value should be a dummy. */ - if (value != Py_None) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto raise_error; - } - /* Normalize to raise , */ - Py_DECREF(value); - value = type; - #if PY_VERSION_HEX < 0x02050000 - if (PyInstance_Check(type)) { - type = (PyObject*) ((PyInstanceObject*)type)->in_class; - Py_INCREF(type); - } - else { - type = 0; - PyErr_SetString(PyExc_TypeError, - "raise: exception must be an old-style class or instance"); - goto raise_error; - } - #else - type = (PyObject*) Py_TYPE(type); - Py_INCREF(type); - if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto raise_error; - } - #endif - } - __Pyx_ErrRestore(type, value, tb); - return; -raise_error: - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(tb); - return; -} - -static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyThreadState *tstate = PyThreadState_GET(); - -#if PY_MAJOR_VERSION >= 3 - /* Note: this is a temporary work-around to prevent crashes in Python 3.0 */ - if ((tstate->exc_type != NULL) & (tstate->exc_type != Py_None)) { - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - PyErr_NormalizeException(&type, &value, &tb); - PyErr_NormalizeException(&tmp_type, &tmp_value, &tmp_tb); - tstate->exc_type = 0; - tstate->exc_value = 0; - tstate->exc_traceback = 0; - PyException_SetContext(value, tmp_value); - Py_DECREF(tmp_type); - Py_XDECREF(tmp_tb); - } -#endif - - tmp_type = tstate->curexc_type; - tmp_value = tstate->curexc_value; - tmp_tb = tstate->curexc_traceback; - tstate->curexc_type = type; - tstate->curexc_value = value; - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -} - -static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb) { - PyThreadState *tstate = PyThreadState_GET(); - *type = tstate->curexc_type; - *value = tstate->curexc_value; - *tb = tstate->curexc_traceback; - - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; -} - - -static INLINE int __Pyx_StrEq(const char *s1, const char *s2) { - while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } - return *s1 == *s2; -} - -static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject* x) { - if (sizeof(unsigned char) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(unsigned char)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (unsigned char)-1; - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned char"); - return (unsigned char)-1; - } - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to unsigned char"); - return (unsigned char)-1; - } - return (unsigned char)val; - } - return (unsigned char)__Pyx_PyInt_AsUnsignedLong(x); -} - -static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject* x) { - if (sizeof(unsigned short) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(unsigned short)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (unsigned short)-1; - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned short"); - return (unsigned short)-1; - } - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to unsigned short"); - return (unsigned short)-1; - } - return (unsigned short)val; - } - return (unsigned short)__Pyx_PyInt_AsUnsignedLong(x); -} - -static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject* x) { - if (sizeof(unsigned int) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(unsigned int)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (unsigned int)-1; - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned int"); - return (unsigned int)-1; - } - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to unsigned int"); - return (unsigned int)-1; - } - return (unsigned int)val; - } - return (unsigned int)__Pyx_PyInt_AsUnsignedLong(x); -} - -static INLINE char __Pyx_PyInt_AsChar(PyObject* x) { - if (sizeof(char) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(char)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (char)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to char"); - return (char)-1; - } - return (char)val; - } - return (char)__Pyx_PyInt_AsLong(x); -} - -static INLINE short __Pyx_PyInt_AsShort(PyObject* x) { - if (sizeof(short) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(short)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (short)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to short"); - return (short)-1; - } - return (short)val; - } - return (short)__Pyx_PyInt_AsLong(x); -} - -static INLINE int __Pyx_PyInt_AsInt(PyObject* x) { - if (sizeof(int) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(int)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (int)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to int"); - return (int)-1; - } - return (int)val; - } - return (int)__Pyx_PyInt_AsLong(x); -} - -static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject* x) { - if (sizeof(signed char) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(signed char)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (signed char)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to signed char"); - return (signed char)-1; - } - return (signed char)val; - } - return (signed char)__Pyx_PyInt_AsSignedLong(x); -} - -static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject* x) { - if (sizeof(signed short) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(signed short)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (signed short)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to signed short"); - return (signed short)-1; - } - return (signed short)val; - } - return (signed short)__Pyx_PyInt_AsSignedLong(x); -} - -static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject* x) { - if (sizeof(signed int) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(signed int)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (signed int)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to signed int"); - return (signed int)-1; - } - return (signed int)val; - } - return (signed int)__Pyx_PyInt_AsSignedLong(x); -} - -static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned long"); - return (unsigned long)-1; - } - return (unsigned long)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - if (unlikely(Py_SIZE(x) < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned long"); - return (unsigned long)-1; - } - return PyLong_AsUnsignedLong(x); - } else { - unsigned long val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (unsigned long)-1; - val = __Pyx_PyInt_AsUnsignedLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned PY_LONG_LONG"); - return (unsigned PY_LONG_LONG)-1; - } - return (unsigned PY_LONG_LONG)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - if (unlikely(Py_SIZE(x) < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned PY_LONG_LONG"); - return (unsigned PY_LONG_LONG)-1; - } - return PyLong_AsUnsignedLongLong(x); - } else { - unsigned PY_LONG_LONG val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (unsigned PY_LONG_LONG)-1; - val = __Pyx_PyInt_AsUnsignedLongLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE long __Pyx_PyInt_AsLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (long)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLong(x); - } else { - long val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (long)-1; - val = __Pyx_PyInt_AsLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (PY_LONG_LONG)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLongLong(x); - } else { - PY_LONG_LONG val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (PY_LONG_LONG)-1; - val = __Pyx_PyInt_AsLongLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (signed long)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLong(x); - } else { - signed long val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (signed long)-1; - val = __Pyx_PyInt_AsSignedLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (signed PY_LONG_LONG)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLongLong(x); - } else { - signed PY_LONG_LONG val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (signed PY_LONG_LONG)-1; - val = __Pyx_PyInt_AsSignedLongLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static void __Pyx_WriteUnraisable(const char *name) { - PyObject *old_exc, *old_val, *old_tb; - PyObject *ctx; - __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); - #if PY_MAJOR_VERSION < 3 - ctx = PyString_FromString(name); - #else - ctx = PyUnicode_FromString(name); - #endif - __Pyx_ErrRestore(old_exc, old_val, old_tb); - if (!ctx) { - PyErr_WriteUnraisable(Py_None); - } else { - PyErr_WriteUnraisable(ctx); - Py_DECREF(ctx); - } -} - -static int __Pyx_SetVtable(PyObject *dict, void *vtable) { - PyObject *pycobj = 0; - int result; - - pycobj = PyCObject_FromVoidPtr(vtable, 0); - if (!pycobj) - goto bad; - if (PyDict_SetItemString(dict, "__pyx_vtable__", pycobj) < 0) - goto bad; - result = 0; - goto done; - -bad: - result = -1; -done: - Py_XDECREF(pycobj); - return result; -} - -#include "compile.h" -#include "frameobject.h" -#include "traceback.h" - -static void __Pyx_AddTraceback(const char *funcname) { - PyObject *py_srcfile = 0; - PyObject *py_funcname = 0; - PyObject *py_globals = 0; - PyObject *empty_string = 0; - PyCodeObject *py_code = 0; - PyFrameObject *py_frame = 0; - - #if PY_MAJOR_VERSION < 3 - py_srcfile = PyString_FromString(__pyx_filename); - #else - py_srcfile = PyUnicode_FromString(__pyx_filename); - #endif - if (!py_srcfile) goto bad; - if (__pyx_clineno) { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); - #else - py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); - #endif - } - else { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromString(funcname); - #else - py_funcname = PyUnicode_FromString(funcname); - #endif - } - if (!py_funcname) goto bad; - py_globals = PyModule_GetDict(__pyx_m); - if (!py_globals) goto bad; - #if PY_MAJOR_VERSION < 3 - empty_string = PyString_FromStringAndSize("", 0); - #else - empty_string = PyBytes_FromStringAndSize("", 0); - #endif - if (!empty_string) goto bad; - py_code = PyCode_New( - 0, /*int argcount,*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*int kwonlyargcount,*/ - #endif - 0, /*int nlocals,*/ - 0, /*int stacksize,*/ - 0, /*int flags,*/ - empty_string, /*PyObject *code,*/ - __pyx_empty_tuple, /*PyObject *consts,*/ - __pyx_empty_tuple, /*PyObject *names,*/ - __pyx_empty_tuple, /*PyObject *varnames,*/ - __pyx_empty_tuple, /*PyObject *freevars,*/ - __pyx_empty_tuple, /*PyObject *cellvars,*/ - py_srcfile, /*PyObject *filename,*/ - py_funcname, /*PyObject *name,*/ - __pyx_lineno, /*int firstlineno,*/ - empty_string /*PyObject *lnotab*/ - ); - if (!py_code) goto bad; - py_frame = PyFrame_New( - PyThreadState_GET(), /*PyThreadState *tstate,*/ - py_code, /*PyCodeObject *code,*/ - py_globals, /*PyObject *globals,*/ - 0 /*PyObject *locals*/ - ); - if (!py_frame) goto bad; - py_frame->f_lineno = __pyx_lineno; - PyTraceBack_Here(py_frame); -bad: - Py_XDECREF(py_srcfile); - Py_XDECREF(py_funcname); - Py_XDECREF(empty_string); - Py_XDECREF(py_code); - Py_XDECREF(py_frame); -} - -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { - while (t->p) { - #if PY_MAJOR_VERSION < 3 - if (t->is_unicode && (!t->is_identifier)) { - *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); - } else if (t->intern) { - *t->p = PyString_InternFromString(t->s); - } else { - *t->p = PyString_FromStringAndSize(t->s, t->n - 1); - } - #else /* Python 3+ has unicode identifiers */ - if (t->is_identifier || (t->is_unicode && t->intern)) { - *t->p = PyUnicode_InternFromString(t->s); - } else if (t->is_unicode) { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } else { - *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); - } - #endif - if (!*t->p) - return -1; - ++t; - } - return 0; -} - -/* Type Conversion Functions */ - -static INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { - if (x == Py_True) return 1; - else if ((x == Py_False) | (x == Py_None)) return 0; - else return PyObject_IsTrue(x); -} - -static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { - PyNumberMethods *m; - const char *name = NULL; - PyObject *res = NULL; -#if PY_VERSION_HEX < 0x03000000 - if (PyInt_Check(x) || PyLong_Check(x)) -#else - if (PyLong_Check(x)) -#endif - return Py_INCREF(x), x; - m = Py_TYPE(x)->tp_as_number; -#if PY_VERSION_HEX < 0x03000000 - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Int(x); - } - else if (m && m->nb_long) { - name = "long"; - res = PyNumber_Long(x); - } -#else - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Long(x); - } -#endif - if (res) { -#if PY_VERSION_HEX < 0x03000000 - if (!PyInt_Check(res) && !PyLong_Check(res)) { -#else - if (!PyLong_Check(res)) { -#endif - PyErr_Format(PyExc_TypeError, - "__%s__ returned non-%s (type %.200s)", - name, name, Py_TYPE(res)->tp_name); - Py_DECREF(res); - return NULL; - } - } - else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, - "an integer is required"); - } - return res; -} - -static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { - Py_ssize_t ival; - PyObject* x = PyNumber_Index(b); - if (!x) return -1; - ival = PyInt_AsSsize_t(x); - Py_DECREF(x); - return ival; -} - -static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { -#if PY_VERSION_HEX < 0x02050000 - if (ival <= LONG_MAX) - return PyInt_FromLong((long)ival); - else { - unsigned char *bytes = (unsigned char *) &ival; - int one = 1; int little = (int)*(unsigned char*)&one; - return _PyLong_FromByteArray(bytes, sizeof(size_t), little, 0); - } -#else - return PyInt_FromSize_t(ival); -#endif -} - -static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject* x) { - unsigned PY_LONG_LONG val = __Pyx_PyInt_AsUnsignedLongLong(x); - if (unlikely(val == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())) { - return (size_t)-1; - } else if (unlikely(val != (unsigned PY_LONG_LONG)(size_t)val)) { - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to size_t"); - return (size_t)-1; - } - return (size_t)val; -} - - diff --git a/python/msgpack/__init__.py b/python/msgpack/__init__.py new file mode 100644 index 0000000..797b29c --- /dev/null +++ b/python/msgpack/__init__.py @@ -0,0 +1,3 @@ +# coding: utf-8 +from _msgpack import * + diff --git a/python/msgpack.pyx b/python/msgpack/_msgpack.pyx similarity index 100% rename from python/msgpack.pyx rename to python/msgpack/_msgpack.pyx diff --git a/python/pack.h b/python/msgpack/pack.h similarity index 100% rename from python/pack.h rename to python/msgpack/pack.h diff --git a/python/unpack.h b/python/msgpack/unpack.h similarity index 100% rename from python/unpack.h rename to python/msgpack/unpack.h diff --git a/python/setup.py b/python/setup.py index 65ca412..56b3faa 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,12 +1,13 @@ from distutils.core import setup, Extension +from Cython.Distutils import build_ext import os version = '0.0.1' PACKAGE_ROOT = os.getcwdu() INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') -msgpack_mod = Extension('msgpack', - sources=['msgpack.c'], +msgpack_mod = Extension('msgpack._msgpack', + sources=['msgpack/_msgpack.pyx'], include_dirs=[INCLUDE_PATH]) desc = 'MessagePack serializer/desirializer.' @@ -28,6 +29,7 @@ setup(name='msgpack', author='Naoki INADA', author_email='songofacandy@gmail.com', version=version, + cmdclass={'build_ext': build_ext}, ext_modules=[msgpack_mod], description=desc, long_description=long_desc, From 74a5fb3592d363092ab73bd90de6ab1d550edb2b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 10:58:09 +0900 Subject: [PATCH 0154/1172] Make msgpack package instead of module. --- MANIFEST | 4 +- msgpack.c | 3438 --------------------------- msgpack/__init__.py | 3 + msgpack.pyx => msgpack/_msgpack.pyx | 0 pack.h => msgpack/pack.h | 0 unpack.h => msgpack/unpack.h | 0 setup.py | 6 +- 7 files changed, 9 insertions(+), 3442 deletions(-) delete mode 100644 msgpack.c create mode 100644 msgpack/__init__.py rename msgpack.pyx => msgpack/_msgpack.pyx (100%) rename pack.h => msgpack/pack.h (100%) rename unpack.h => msgpack/unpack.h (100%) diff --git a/MANIFEST b/MANIFEST index dc042ae..f2da7da 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1,7 +1,7 @@ msgpack.c setup.py -pack.h -unpack.h +msgpack/pack.h +msgpack/unpack.h include/msgpack/pack_define.h include/msgpack/pack_template.h include/msgpack/unpack_define.h diff --git a/msgpack.c b/msgpack.c deleted file mode 100644 index 821f65b..0000000 --- a/msgpack.c +++ /dev/null @@ -1,3438 +0,0 @@ -/* Generated by Cython 0.11.2 on Tue Jun 9 13:10:11 2009 */ - -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#include "structmember.h" -#ifndef Py_PYTHON_H - #error Python headers needed to compile C extensions, please install development version of Python. -#endif -#ifndef PY_LONG_LONG - #define PY_LONG_LONG LONG_LONG -#endif -#ifndef DL_EXPORT - #define DL_EXPORT(t) t -#endif -#if PY_VERSION_HEX < 0x02040000 - #define METH_COEXIST 0 - #define PyDict_CheckExact(op) (Py_TYPE(op) == &PyDict_Type) -#endif -#if PY_VERSION_HEX < 0x02050000 - typedef int Py_ssize_t; - #define PY_SSIZE_T_MAX INT_MAX - #define PY_SSIZE_T_MIN INT_MIN - #define PY_FORMAT_SIZE_T "" - #define PyInt_FromSsize_t(z) PyInt_FromLong(z) - #define PyInt_AsSsize_t(o) PyInt_AsLong(o) - #define PyNumber_Index(o) PyNumber_Int(o) - #define PyIndex_Check(o) PyNumber_Check(o) -#endif -#if PY_VERSION_HEX < 0x02060000 - #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) - #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) - #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) - #define PyVarObject_HEAD_INIT(type, size) \ - PyObject_HEAD_INIT(type) size, - #define PyType_Modified(t) - - typedef struct { - void *buf; - PyObject *obj; - Py_ssize_t len; - Py_ssize_t itemsize; - int readonly; - int ndim; - char *format; - Py_ssize_t *shape; - Py_ssize_t *strides; - Py_ssize_t *suboffsets; - void *internal; - } Py_buffer; - - #define PyBUF_SIMPLE 0 - #define PyBUF_WRITABLE 0x0001 - #define PyBUF_FORMAT 0x0004 - #define PyBUF_ND 0x0008 - #define PyBUF_STRIDES (0x0010 | PyBUF_ND) - #define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES) - #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES) - #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES) - #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES) - -#endif -#if PY_MAJOR_VERSION < 3 - #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" -#else - #define __Pyx_BUILTIN_MODULE_NAME "builtins" -#endif -#if PY_MAJOR_VERSION >= 3 - #define Py_TPFLAGS_CHECKTYPES 0 - #define Py_TPFLAGS_HAVE_INDEX 0 -#endif -#if (PY_VERSION_HEX < 0x02060000) || (PY_MAJOR_VERSION >= 3) - #define Py_TPFLAGS_HAVE_NEWBUFFER 0 -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBaseString_Type PyUnicode_Type - #define PyString_Type PyBytes_Type - #define PyString_CheckExact PyBytes_CheckExact - #define PyInt_Type PyLong_Type - #define PyInt_Check(op) PyLong_Check(op) - #define PyInt_CheckExact(op) PyLong_CheckExact(op) - #define PyInt_FromString PyLong_FromString - #define PyInt_FromUnicode PyLong_FromUnicode - #define PyInt_FromLong PyLong_FromLong - #define PyInt_FromSize_t PyLong_FromSize_t - #define PyInt_FromSsize_t PyLong_FromSsize_t - #define PyInt_AsLong PyLong_AsLong - #define PyInt_AS_LONG PyLong_AS_LONG - #define PyInt_AsSsize_t PyLong_AsSsize_t - #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask - #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask - #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) -#else - #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) - #define PyBytes_Type PyString_Type -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyMethod_New(func, self, klass) PyInstanceMethod_New(func) -#endif -#if !defined(WIN32) && !defined(MS_WINDOWS) - #ifndef __stdcall - #define __stdcall - #endif - #ifndef __cdecl - #define __cdecl - #endif - #ifndef __fastcall - #define __fastcall - #endif -#else - #define _USE_MATH_DEFINES -#endif -#if PY_VERSION_HEX < 0x02050000 - #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),((char *)(n))) - #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),((char *)(n)),(a)) - #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),((char *)(n))) -#else - #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),(n)) - #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),(n),(a)) - #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),(n)) -#endif -#if PY_VERSION_HEX < 0x02050000 - #define __Pyx_NAMESTR(n) ((char *)(n)) - #define __Pyx_DOCSTR(n) ((char *)(n)) -#else - #define __Pyx_NAMESTR(n) (n) - #define __Pyx_DOCSTR(n) (n) -#endif -#ifdef __cplusplus -#define __PYX_EXTERN_C extern "C" -#else -#define __PYX_EXTERN_C extern -#endif -#include -#define __PYX_HAVE_API__msgpack -#include "stdlib.h" -#include "string.h" -#include "pack.h" -#include "unpack.h" -#define __PYX_USE_C99_COMPLEX defined(_Complex_I) - - -#ifdef __GNUC__ -#define INLINE __inline__ -#elif _WIN32 -#define INLINE __inline -#else -#define INLINE -#endif - -typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern; char is_identifier;} __Pyx_StringTabEntry; /*proto*/ - - - -static int __pyx_skip_dispatch = 0; - - -/* Type Conversion Predeclarations */ - -#if PY_MAJOR_VERSION < 3 -#define __Pyx_PyBytes_FromString PyString_FromString -#define __Pyx_PyBytes_FromStringAndSize PyString_FromStringAndSize -#define __Pyx_PyBytes_AsString PyString_AsString -#else -#define __Pyx_PyBytes_FromString PyBytes_FromString -#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize -#define __Pyx_PyBytes_AsString PyBytes_AsString -#endif - -#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False)) -static INLINE int __Pyx_PyObject_IsTrue(PyObject*); -static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x); - -#if !defined(T_PYSSIZET) -#if PY_VERSION_HEX < 0x02050000 -#define T_PYSSIZET T_INT -#elif !defined(T_LONGLONG) -#define T_PYSSIZET \ - ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ - ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : -1)) -#else -#define T_PYSSIZET \ - ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ - ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : \ - ((sizeof(Py_ssize_t) == sizeof(PY_LONG_LONG)) ? T_LONGLONG : -1))) -#endif -#endif - -#if !defined(T_SIZET) -#if !defined(T_ULONGLONG) -#define T_SIZET \ - ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ - ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : -1)) -#else -#define T_SIZET \ - ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ - ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : \ - ((sizeof(size_t) == sizeof(unsigned PY_LONG_LONG)) ? T_ULONGLONG : -1))) -#endif -#endif - -static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); -static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); -static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*); - -#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) - - -#ifdef __GNUC__ -/* Test for GCC > 2.95 */ -#if __GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)) -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) -#else /* __GNUC__ > 2 ... */ -#define likely(x) (x) -#define unlikely(x) (x) -#endif /* __GNUC__ > 2 ... */ -#else /* __GNUC__ */ -#define likely(x) (x) -#define unlikely(x) (x) -#endif /* __GNUC__ */ - -static PyObject *__pyx_m; -static PyObject *__pyx_b; -static PyObject *__pyx_empty_tuple; -static int __pyx_lineno; -static int __pyx_clineno = 0; -static const char * __pyx_cfilenm= __FILE__; -static const char *__pyx_filename; -static const char **__pyx_f; - - -#ifdef CYTHON_REFNANNY -typedef struct { - void (*INCREF)(void*, PyObject*, int); - void (*DECREF)(void*, PyObject*, int); - void (*GOTREF)(void*, PyObject*, int); - void (*GIVEREF)(void*, PyObject*, int); - void* (*NewContext)(const char*, int, const char*); - void (*FinishContext)(void**); -} __Pyx_RefnannyAPIStruct; -static __Pyx_RefnannyAPIStruct *__Pyx_Refnanny = NULL; -#define __Pyx_ImportRefcountAPI(name) (__Pyx_RefnannyAPIStruct *) PyCObject_Import((char *)name, (char *)"RefnannyAPI") -#define __Pyx_INCREF(r) __Pyx_Refnanny->INCREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_DECREF(r) __Pyx_Refnanny->DECREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_GOTREF(r) __Pyx_Refnanny->GOTREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_GIVEREF(r) __Pyx_Refnanny->GIVEREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_XDECREF(r) if((r) == NULL) ; else __Pyx_DECREF(r) -#define __Pyx_SetupRefcountContext(name) void* __pyx_refchk = __Pyx_Refnanny->NewContext((name), __LINE__, __FILE__) -#define __Pyx_FinishRefcountContext() __Pyx_Refnanny->FinishContext(&__pyx_refchk) -#else -#define __Pyx_INCREF(r) Py_INCREF(r) -#define __Pyx_DECREF(r) Py_DECREF(r) -#define __Pyx_GOTREF(r) -#define __Pyx_GIVEREF(r) -#define __Pyx_XDECREF(r) Py_XDECREF(r) -#define __Pyx_SetupRefcountContext(name) -#define __Pyx_FinishRefcountContext() -#endif /* CYTHON_REFNANNY */ -#define __Pyx_XGIVEREF(r) if((r) == NULL) ; else __Pyx_GIVEREF(r) -#define __Pyx_XGOTREF(r) if((r) == NULL) ; else __Pyx_GOTREF(r) - -static void __Pyx_RaiseDoubleKeywordsError( - const char* func_name, PyObject* kw_name); /*proto*/ - -static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, - Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); /*proto*/ - -static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, const char* function_name); /*proto*/ - -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list); /*proto*/ - -static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name); /*proto*/ - -static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); - -static INLINE void __Pyx_RaiseTooManyValuesError(void); - -static PyObject *__Pyx_UnpackItem(PyObject *, Py_ssize_t index); /*proto*/ -static int __Pyx_EndUnpack(PyObject *); /*proto*/ - -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ - -static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ -static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ - -static INLINE int __Pyx_StrEq(const char *, const char *); /*proto*/ - -static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject *); - -static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject *); - -static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject *); - -static INLINE char __Pyx_PyInt_AsChar(PyObject *); - -static INLINE short __Pyx_PyInt_AsShort(PyObject *); - -static INLINE int __Pyx_PyInt_AsInt(PyObject *); - -static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject *); - -static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject *); - -static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject *); - -static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject *); - -static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject *); - -static INLINE long __Pyx_PyInt_AsLong(PyObject *); - -static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject *); - -static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject *); - -static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); - -static void __Pyx_WriteUnraisable(const char *name); /*proto*/ - -static int __Pyx_SetVtable(PyObject *dict, void *vtable); /*proto*/ - -static void __Pyx_AddTraceback(const char *funcname); /*proto*/ - -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ - -/* Type declarations */ - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 - * cdef int BUFF_SIZE=2*1024 - * - * cdef class Packer: # <<<<<<<<<<<<<< - * """Packer that pack data into strm. - * - */ - -struct __pyx_obj_7msgpack_Packer { - PyObject_HEAD - struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtab; - char *buff; - unsigned int length; - unsigned int allocated; - struct msgpack_packer pk; - PyObject *strm; -}; - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 - * return unpacks(packed) - * - * cdef class Unpacker: # <<<<<<<<<<<<<< - * """Do nothing. This function is for symmetric to Packer""" - * unpack = staticmethod(unpacks) - */ - -struct __pyx_obj_7msgpack_Unpacker { - PyObject_HEAD -}; - - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 - * cdef int BUFF_SIZE=2*1024 - * - * cdef class Packer: # <<<<<<<<<<<<<< - * """Packer that pack data into strm. - * - */ - -struct __pyx_vtabstruct_7msgpack_Packer { - PyObject *(*__pack)(struct __pyx_obj_7msgpack_Packer *, PyObject *); -}; -static struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtabptr_7msgpack_Packer; -/* Module declarations from msgpack */ - -static PyTypeObject *__pyx_ptype_7msgpack_Packer = 0; -static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; -static int __pyx_v_7msgpack_BUFF_SIZE; -static PyObject *__pyx_k_1 = 0; -static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ -#define __Pyx_MODULE_NAME "msgpack" -int __pyx_module_is_main_msgpack = 0; - -/* Implementation of msgpack */ -static char __pyx_k___main__[] = "__main__"; -static PyObject *__pyx_kp___main__; -static char __pyx_k___init__[] = "__init__"; -static PyObject *__pyx_kp___init__; -static char __pyx_k___del__[] = "__del__"; -static PyObject *__pyx_kp___del__; -static char __pyx_k_flush[] = "flush"; -static PyObject *__pyx_kp_flush; -static char __pyx_k_pack_list[] = "pack_list"; -static PyObject *__pyx_kp_pack_list; -static char __pyx_k_pack_dict[] = "pack_dict"; -static PyObject *__pyx_kp_pack_dict; -static char __pyx_k_pack[] = "pack"; -static PyObject *__pyx_kp_pack; -static char __pyx_k_unpack[] = "unpack"; -static PyObject *__pyx_kp_unpack; -static char __pyx_k_strm[] = "strm"; -static PyObject *__pyx_kp_strm; -static char __pyx_k_size[] = "size"; -static PyObject *__pyx_kp_size; -static char __pyx_k_len[] = "len"; -static PyObject *__pyx_kp_len; -static char __pyx_k_obj[] = "obj"; -static PyObject *__pyx_kp_obj; -static char __pyx_k_o[] = "o"; -static PyObject *__pyx_kp_o; -static char __pyx_k_stream[] = "stream"; -static PyObject *__pyx_kp_stream; -static char __pyx_k_packed_bytes[] = "packed_bytes"; -static PyObject *__pyx_kp_packed_bytes; -static char __pyx_k_cStringIO[] = "cStringIO"; -static PyObject *__pyx_kp_cStringIO; -static char __pyx_k_StringIO[] = "StringIO"; -static PyObject *__pyx_kp_StringIO; -static char __pyx_k_staticmethod[] = "staticmethod"; -static PyObject *__pyx_kp_staticmethod; -static char __pyx_k_unpacks[] = "unpacks"; -static PyObject *__pyx_kp_unpacks; -static char __pyx_k_write[] = "write"; -static PyObject *__pyx_kp_write; -static char __pyx_k_2[] = "flush"; -static PyObject *__pyx_kp_2; -static char __pyx_k_encode[] = "encode"; -static PyObject *__pyx_kp_encode; -static char __pyx_k_iteritems[] = "iteritems"; -static PyObject *__pyx_kp_iteritems; -static char __pyx_k_TypeError[] = "TypeError"; -static PyObject *__pyx_kp_TypeError; -static char __pyx_k_getvalue[] = "getvalue"; -static PyObject *__pyx_kp_getvalue; -static char __pyx_k_read[] = "read"; -static PyObject *__pyx_kp_read; -static PyObject *__pyx_builtin_staticmethod; -static PyObject *__pyx_builtin_TypeError; -static PyObject *__pyx_kp_3; -static PyObject *__pyx_kp_4; -static char __pyx_k_3[] = "utf-8"; -static char __pyx_k_4[] = "can't serialize %r"; - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 - * cdef object strm - * - * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< - * if size <= 0: - * size = BUFF_SIZE - */ - -static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_strm = 0; - int __pyx_v_size; - int __pyx_r; - int __pyx_t_1; - static PyObject **__pyx_pyargnames[] = {&__pyx_kp_strm,&__pyx_kp_size,0}; - __Pyx_SetupRefcountContext("__init__"); - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); - PyObject* values[2] = {0,0}; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 0: - values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_strm); - if (likely(values[0])) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (kw_args > 1) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_size); - if (unlikely(value)) { values[1] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "__init__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - __pyx_v_strm = values[0]; - if (values[1]) { - __pyx_v_size = __Pyx_PyInt_AsInt(values[1]); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } else { - __pyx_v_size = 0; - } - } else { - __pyx_v_size = 0; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: __pyx_v_size = __Pyx_PyInt_AsInt(PyTuple_GET_ITEM(__pyx_args, 1)); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - case 1: __pyx_v_strm = PyTuple_GET_ITEM(__pyx_args, 0); - break; - default: goto __pyx_L5_argtuple_error; - } - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack.Packer.__init__"); - return -1; - __pyx_L4_argument_unpacking_done:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 - * - * def __init__(self, strm, int size=0): - * if size <= 0: # <<<<<<<<<<<<<< - * size = BUFF_SIZE - * - */ - __pyx_t_1 = (__pyx_v_size <= 0); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 - * def __init__(self, strm, int size=0): - * if size <= 0: - * size = BUFF_SIZE # <<<<<<<<<<<<<< - * - * self.strm = strm - */ - __pyx_v_size = __pyx_v_7msgpack_BUFF_SIZE; - goto __pyx_L6; - } - __pyx_L6:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 - * size = BUFF_SIZE - * - * self.strm = strm # <<<<<<<<<<<<<< - * self.buff = malloc(size) - * self.allocated = size - */ - __Pyx_INCREF(__pyx_v_strm); - __Pyx_GIVEREF(__pyx_v_strm); - __Pyx_GOTREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); - __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 - * - * self.strm = strm - * self.buff = malloc(size) # <<<<<<<<<<<<<< - * self.allocated = size - * self.length = 0 - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 - * self.strm = strm - * self.buff = malloc(size) - * self.allocated = size # <<<<<<<<<<<<<< - * self.length = 0 - * - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 - * self.buff = malloc(size) - * self.allocated = size - * self.length = 0 # <<<<<<<<<<<<<< - * - * msgpack_packer_init(&self.pk, self, _packer_write) - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 - * self.length = 0 - * - * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< - * - * def __del__(self): - */ - msgpack_packer_init((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), ((void *)__pyx_v_self), ((int (*)(void *, const char*, unsigned int))__pyx_f_7msgpack__packer_write)); - - __pyx_r = 0; - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 - * msgpack_packer_init(&self.pk, self, _packer_write) - * - * def __del__(self): # <<<<<<<<<<<<<< - * free(self.buff); - * - */ - -static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ -static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused) { - PyObject *__pyx_r = NULL; - __Pyx_SetupRefcountContext("__del__"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 - * - * def __del__(self): - * free(self.buff); # <<<<<<<<<<<<<< - * - * def flush(self): - */ - free(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff); - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 - * free(self.buff); - * - * def flush(self): # <<<<<<<<<<<<<< - * """Flash local buffer and output stream if it has 'flush()' method.""" - * if self.length > 0: - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ -static char __pyx_doc_7msgpack_6Packer_flush[] = "Flash local buffer and output stream if it has 'flush()' method."; -static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused) { - PyObject *__pyx_r = NULL; - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - __Pyx_SetupRefcountContext("flush"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 - * def flush(self): - * """Flash local buffer and output stream if it has 'flush()' method.""" - * if self.length > 0: # <<<<<<<<<<<<<< - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - * self.length = 0 - */ - __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 - * """Flash local buffer and output stream if it has 'flush()' method.""" - * if self.length > 0: - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< - * self.length = 0 - * if hasattr(self.strm, 'flush'): - */ - __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_4)); - PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_3); - __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 - * if self.length > 0: - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - * self.length = 0 # <<<<<<<<<<<<<< - * if hasattr(self.strm, 'flush'): - * self.strm.flush() - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - goto __pyx_L5; - } - __pyx_L5:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - * self.length = 0 - * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< - * self.strm.flush() - * - */ - __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 - * self.length = 0 - * if hasattr(self.strm, 'flush'): - * self.strm.flush() # <<<<<<<<<<<<<< - * - * def pack_list(self, len): - */ - __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - goto __pyx_L6; - } - __pyx_L6:; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("msgpack.Packer.flush"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 - * self.strm.flush() - * - * def pack_list(self, len): # <<<<<<<<<<<<<< - * """Start packing sequential objects. - * - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ -static char __pyx_doc_7msgpack_6Packer_pack_list[] = "Start packing sequential objects.\n\n Example:\n\n packer.pack_list(2)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack(['foo', 'bar'])\n "; -static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { - PyObject *__pyx_r = NULL; - size_t __pyx_t_1; - __Pyx_SetupRefcountContext("pack_list"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 - * packer.pack(['foo', 'bar']) - * """ - * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< - * - * def pack_dict(self, len): - */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_AddTraceback("msgpack.Packer.pack_list"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 - * msgpack_pack_array(&self.pk, len) - * - * def pack_dict(self, len): # <<<<<<<<<<<<<< - * """Start packing key-value objects. - * - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ -static char __pyx_doc_7msgpack_6Packer_pack_dict[] = "Start packing key-value objects.\n\n Example:\n\n packer.pack_dict(1)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack({'foo', 'bar'})\n "; -static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { - PyObject *__pyx_r = NULL; - size_t __pyx_t_1; - __Pyx_SetupRefcountContext("pack_dict"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 - * packer.pack({'foo', 'bar'}) - * """ - * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< - * - * cdef __pack(self, object o): - */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_AddTraceback("msgpack.Packer.pack_dict"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 - * msgpack_pack_map(&self.pk, len) - * - * cdef __pack(self, object o): # <<<<<<<<<<<<<< - * cdef long long intval - * cdef double fval - */ - -static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Packer *__pyx_v_self, PyObject *__pyx_v_o) { - PY_LONG_LONG __pyx_v_intval; - double __pyx_v_fval; - char *__pyx_v_rawval; - PyObject *__pyx_v_k; - PyObject *__pyx_v_v; - PyObject *__pyx_r = NULL; - PyObject *__pyx_1 = 0; - PyObject *__pyx_2 = 0; - PyObject *__pyx_3 = 0; - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - PY_LONG_LONG __pyx_t_3; - char *__pyx_t_4; - Py_ssize_t __pyx_t_5; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - PyObject *__pyx_t_8 = NULL; - int __pyx_t_9; - int __pyx_t_10; - __Pyx_SetupRefcountContext("__pack"); - __Pyx_INCREF(__pyx_v_o); - __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); - __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 - * cdef char* rawval - * - * if o is None: # <<<<<<<<<<<<<< - * msgpack_pack_nil(&self.pk) - * elif o is True: - */ - __pyx_t_1 = (__pyx_v_o == Py_None); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 - * - * if o is None: - * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< - * elif o is True: - * msgpack_pack_true(&self.pk) - */ - msgpack_pack_nil((&__pyx_v_self->pk)); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 - * if o is None: - * msgpack_pack_nil(&self.pk) - * elif o is True: # <<<<<<<<<<<<<< - * msgpack_pack_true(&self.pk) - * elif o is False: - */ - __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = (__pyx_v_o == __pyx_t_2); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 - * msgpack_pack_nil(&self.pk) - * elif o is True: - * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< - * elif o is False: - * msgpack_pack_false(&self.pk) - */ - msgpack_pack_true((&__pyx_v_self->pk)); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 - * elif o is True: - * msgpack_pack_true(&self.pk) - * elif o is False: # <<<<<<<<<<<<<< - * msgpack_pack_false(&self.pk) - * elif isinstance(o, long): - */ - __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = (__pyx_v_o == __pyx_t_2); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 - * msgpack_pack_true(&self.pk) - * elif o is False: - * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< - * elif isinstance(o, long): - * intval = o - */ - msgpack_pack_false((&__pyx_v_self->pk)); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 - * elif o is False: - * msgpack_pack_false(&self.pk) - * elif isinstance(o, long): # <<<<<<<<<<<<<< - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 - * msgpack_pack_false(&self.pk) - * elif isinstance(o, long): - * intval = o # <<<<<<<<<<<<<< - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, int): - */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_intval = __pyx_t_3; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 - * elif isinstance(o, long): - * intval = o - * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< - * elif isinstance(o, int): - * intval = o - */ - msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, int): # <<<<<<<<<<<<<< - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, int): - * intval = o # <<<<<<<<<<<<<< - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, float): - */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_intval = __pyx_t_3; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 - * elif isinstance(o, int): - * intval = o - * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< - * elif isinstance(o, float): - * fval = 9 - */ - msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, float): # <<<<<<<<<<<<<< - * fval = 9 - * msgpack_pack_double(&self.pk, fval) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, float): - * fval = 9 # <<<<<<<<<<<<<< - * msgpack_pack_double(&self.pk, fval) - * elif isinstance(o, str): - */ - __pyx_v_fval = 9; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 - * elif isinstance(o, float): - * fval = 9 - * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< - * elif isinstance(o, str): - * rawval = o - */ - msgpack_pack_double((&__pyx_v_self->pk), __pyx_v_fval); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 - * fval = 9 - * msgpack_pack_double(&self.pk, fval) - * elif isinstance(o, str): # <<<<<<<<<<<<<< - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 - * msgpack_pack_double(&self.pk, fval) - * elif isinstance(o, str): - * rawval = o # <<<<<<<<<<<<<< - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 - * elif isinstance(o, str): - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, unicode): - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< - * elif isinstance(o, unicode): - * o = o.encode('utf-8') - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, unicode): # <<<<<<<<<<<<<< - * o = o.encode('utf-8') - * rawval = o - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, unicode): - * o = o.encode('utf-8') # <<<<<<<<<<<<<< - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_6)); - __Pyx_INCREF(__pyx_kp_3); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_3); - __Pyx_GIVEREF(__pyx_kp_3); - __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; - __Pyx_DECREF(__pyx_v_o); - __pyx_v_o = __pyx_t_7; - __pyx_t_7 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 - * elif isinstance(o, unicode): - * o = o.encode('utf-8') - * rawval = o # <<<<<<<<<<<<<< - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 - * o = o.encode('utf-8') - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, dict): - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< - * elif isinstance(o, dict): - * msgpack_pack_map(&self.pk, len(o)) - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, dict): # <<<<<<<<<<<<<< - * msgpack_pack_map(&self.pk, len(o)) - * for k,v in o.iteritems(): - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, dict): - * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< - * for k,v in o.iteritems(): - * self.pack(k) - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 - * elif isinstance(o, dict): - * msgpack_pack_map(&self.pk, len(o)) - * for k,v in o.iteritems(): # <<<<<<<<<<<<<< - * self.pack(k) - * self.pack(v) - */ - __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); - } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - } - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - break; - } - __Pyx_GOTREF(__pyx_t_6); - } - if (PyTuple_CheckExact(__pyx_t_6) && likely(PyTuple_GET_SIZE(__pyx_t_6) == 2)) { - PyObject* tuple = __pyx_t_6; - __pyx_2 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_2); - __pyx_3 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_3); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(__pyx_v_k); - __pyx_v_k = __pyx_2; - __pyx_2 = 0; - __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_3; - __pyx_3 = 0; - } else { - __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_2); - __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_3); - if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - __Pyx_DECREF(__pyx_v_k); - __pyx_v_k = __pyx_2; - __pyx_2 = 0; - __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_3; - __pyx_3 = 0; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 - * msgpack_pack_map(&self.pk, len(o)) - * for k,v in o.iteritems(): - * self.pack(k) # <<<<<<<<<<<<<< - * self.pack(v) - * elif isinstance(o, tuple) or isinstance(o, list): - */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_k); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); - __Pyx_GIVEREF(__pyx_v_k); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 - * for k,v in o.iteritems(): - * self.pack(k) - * self.pack(v) # <<<<<<<<<<<<<< - * elif isinstance(o, tuple) or isinstance(o, list): - * msgpack_pack_array(&self.pk, len(o)) - */ - __pyx_t_8 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_v); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); - __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 - * self.pack(k) - * self.pack(v) - * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< - * msgpack_pack_array(&self.pk, len(o)) - * for v in o: - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyTuple_Type))); - if (!__pyx_t_1) { - __pyx_t_9 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); - __pyx_t_10 = __pyx_t_9; - } else { - __pyx_t_10 = __pyx_t_1; - } - if (__pyx_t_10) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 - * self.pack(v) - * elif isinstance(o, tuple) or isinstance(o, list): - * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< - * for v in o: - * self.pack(v) - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 - * elif isinstance(o, tuple) or isinstance(o, list): - * msgpack_pack_array(&self.pk, len(o)) - * for v in o: # <<<<<<<<<<<<<< - * self.pack(v) - * else: - */ - if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); - } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - } - for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - break; - } - __Pyx_GOTREF(__pyx_t_6); - } - __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_t_6; - __pyx_t_6 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 - * msgpack_pack_array(&self.pk, len(o)) - * for v in o: - * self.pack(v) # <<<<<<<<<<<<<< - * else: - * # TODO: Serialize with defalt() like simplejson. - */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_v); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); - __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L3; - } - /*else*/ { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 - * else: - * # TODO: Serialize with defalt() like simplejson. - * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< - * - * def pack(self, obj, flush=True): - */ - __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_7)); - __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); - __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_8 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; - __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - __pyx_L3:; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_2); - __Pyx_XDECREF(__pyx_3); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_XDECREF(__pyx_t_8); - __Pyx_AddTraceback("msgpack.Packer.__pack"); - __pyx_r = 0; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_k); - __Pyx_DECREF(__pyx_v_v); - __Pyx_DECREF(__pyx_v_o); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 - * raise TypeError, "can't serialize %r" % (o,) - * - * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< - * self.__pack(obj) - * if flush: - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_obj = 0; - PyObject *__pyx_v_flush = 0; - PyObject *__pyx_r = NULL; - PyObject *__pyx_t_1 = NULL; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - static PyObject **__pyx_pyargnames[] = {&__pyx_kp_obj,&__pyx_kp_flush,0}; - __Pyx_SetupRefcountContext("pack"); - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); - PyObject* values[2] = {0,0}; - values[1] = __pyx_k_1; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 0: - values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_obj); - if (likely(values[0])) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (kw_args > 1) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_flush); - if (unlikely(value)) { values[1] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - __pyx_v_obj = values[0]; - __pyx_v_flush = values[1]; - } else { - __pyx_v_flush = __pyx_k_1; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: __pyx_v_flush = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: __pyx_v_obj = PyTuple_GET_ITEM(__pyx_args, 0); - break; - default: goto __pyx_L5_argtuple_error; - } - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("pack", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack.Packer.pack"); - return NULL; - __pyx_L4_argument_unpacking_done:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 - * - * def pack(self, obj, flush=True): - * self.__pack(obj) # <<<<<<<<<<<<<< - * if flush: - * self.flush() - */ - __pyx_t_1 = ((struct __pyx_vtabstruct_7msgpack_Packer *)((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->__pyx_vtab)->__pack(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self), __pyx_v_obj); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 - * def pack(self, obj, flush=True): - * self.__pack(obj) - * if flush: # <<<<<<<<<<<<<< - * self.flush() - * - */ - __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__pyx_t_2) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 - * self.__pack(obj) - * if flush: - * self.flush() # <<<<<<<<<<<<<< - * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - */ - __pyx_t_1 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_flush); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - goto __pyx_L6; - } - __pyx_L6:; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.Packer.pack"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 - * self.flush() - * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< - * if packer.length + l > packer.allocated: - * if packer.length > 0: - */ - -static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__pyx_v_packer, const char* __pyx_v_b, unsigned int __pyx_v_l) { - int __pyx_r; - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - __Pyx_SetupRefcountContext("_packer_write"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 - * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< - * if packer.length > 0: - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - */ - __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - * if packer.length + l > packer.allocated: - * if packer.length > 0: # <<<<<<<<<<<<<< - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - * if l > 64: - */ - __pyx_t_1 = (__pyx_v_packer->length > 0); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 - * if packer.length + l > packer.allocated: - * if packer.length > 0: - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< - * if l > 64: - * packer.strm.write(PyString_FromStringAndSize(b, l)) - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_4)); - PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_3); - __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - goto __pyx_L4; - } - __pyx_L4:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 - * if packer.length > 0: - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - * if l > 64: # <<<<<<<<<<<<<< - * packer.strm.write(PyString_FromStringAndSize(b, l)) - * packer.length = 0 - */ - __pyx_t_1 = (__pyx_v_l > 64); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - * if l > 64: - * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< - * packer.length = 0 - * else: - */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_4); - __Pyx_GIVEREF(__pyx_t_4); - __pyx_t_4 = 0; - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 - * if l > 64: - * packer.strm.write(PyString_FromStringAndSize(b, l)) - * packer.length = 0 # <<<<<<<<<<<<<< - * else: - * memcpy(packer.buff, b, l) - */ - __pyx_v_packer->length = 0; - goto __pyx_L5; - } - /*else*/ { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 - * packer.length = 0 - * else: - * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< - * packer.length = l - * else: - */ - memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 - * else: - * memcpy(packer.buff, b, l) - * packer.length = l # <<<<<<<<<<<<<< - * else: - * memcpy(packer.buff + packer.length, b, l) - */ - __pyx_v_packer->length = __pyx_v_l; - } - __pyx_L5:; - goto __pyx_L3; - } - /*else*/ { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 - * packer.length = l - * else: - * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< - * packer.length += l - * return 0 - */ - memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 - * else: - * memcpy(packer.buff + packer.length, b, l) - * packer.length += l # <<<<<<<<<<<<<< - * return 0 - * - */ - __pyx_v_packer->length += __pyx_v_l; - } - __pyx_L3:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 - * memcpy(packer.buff + packer.length, b, l) - * packer.length += l - * return 0 # <<<<<<<<<<<<<< - * - * def pack(object o, object stream): - */ - __pyx_r = 0; - goto __pyx_L0; - - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_WriteUnraisable("msgpack._packer_write"); - __pyx_r = 0; - __pyx_L0:; - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 - * return 0 - * - * def pack(object o, object stream): # <<<<<<<<<<<<<< - * packer = Packer(stream) - * packer.pack(o) - */ - -static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_o = 0; - PyObject *__pyx_v_stream = 0; - PyObject *__pyx_v_packer; - PyObject *__pyx_r = NULL; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - static PyObject **__pyx_pyargnames[] = {&__pyx_kp_o,&__pyx_kp_stream,0}; - __Pyx_SetupRefcountContext("pack"); - __pyx_self = __pyx_self; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); - PyObject* values[2] = {0,0}; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 0: - values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_o); - if (likely(values[0])) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - values[1] = PyDict_GetItem(__pyx_kwds, __pyx_kp_stream); - if (likely(values[1])) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - __pyx_v_o = values[0]; - __pyx_v_stream = values[1]; - } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { - goto __pyx_L5_argtuple_error; - } else { - __pyx_v_o = PyTuple_GET_ITEM(__pyx_args, 0); - __pyx_v_stream = PyTuple_GET_ITEM(__pyx_args, 1); - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack.pack"); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 - * - * def pack(object o, object stream): - * packer = Packer(stream) # <<<<<<<<<<<<<< - * packer.pack(o) - * packer.flush() - */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_stream); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_stream); - __Pyx_GIVEREF(__pyx_v_stream); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_v_packer); - __pyx_v_packer = __pyx_t_2; - __pyx_t_2 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 - * def pack(object o, object stream): - * packer = Packer(stream) - * packer.pack(o) # <<<<<<<<<<<<<< - * packer.flush() - * - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); - __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 - * packer = Packer(stream) - * packer.pack(o) - * packer.flush() # <<<<<<<<<<<<<< - * - * def packs(object o): - */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.pack"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_packer); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 - * packer.flush() - * - * def packs(object o): # <<<<<<<<<<<<<< - * buf = StringIO() - * packer = Packer(buf) - */ - -static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o); /*proto*/ -static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o) { - PyObject *__pyx_v_buf; - PyObject *__pyx_v_packer; - PyObject *__pyx_r = NULL; - PyObject *__pyx_1 = 0; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - __Pyx_SetupRefcountContext("packs"); - __pyx_self = __pyx_self; - __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); - __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 - * - * def packs(object o): - * buf = StringIO() # <<<<<<<<<<<<<< - * packer = Packer(buf) - * packer.pack(o) - */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - __Pyx_DECREF(__pyx_v_buf); - __pyx_v_buf = __pyx_t_1; - __pyx_t_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 - * def packs(object o): - * buf = StringIO() - * packer = Packer(buf) # <<<<<<<<<<<<<< - * packer.pack(o) - * packer.flush() - */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_buf); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_buf); - __Pyx_GIVEREF(__pyx_v_buf); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_v_packer); - __pyx_v_packer = __pyx_t_2; - __pyx_t_2 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 - * buf = StringIO() - * packer = Packer(buf) - * packer.pack(o) # <<<<<<<<<<<<<< - * packer.flush() - * return buf.getvalue() - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); - __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 - * packer = Packer(buf) - * packer.pack(o) - * packer.flush() # <<<<<<<<<<<<<< - * return buf.getvalue() - * - */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 - * packer.pack(o) - * packer.flush() - * return buf.getvalue() # <<<<<<<<<<<<<< - * - * cdef extern from "unpack.h": - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_r = __pyx_t_3; - __pyx_t_3 = 0; - goto __pyx_L0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.packs"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_buf); - __Pyx_DECREF(__pyx_v_packer); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 - * - * - * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< - * """Unpack packed_bytes to object. Returns unpacked object.""" - * cdef const_char_ptr p = packed_bytes - */ - -static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes); /*proto*/ -static char __pyx_doc_7msgpack_unpacks[] = "Unpack packed_bytes to object. Returns unpacked object."; -static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes) { - const char* __pyx_v_p; - template_context __pyx_v_ctx; - size_t __pyx_v_off; - PyObject *__pyx_r = NULL; - const char* __pyx_t_1; - Py_ssize_t __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - __Pyx_SetupRefcountContext("unpacks"); - __pyx_self = __pyx_self; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 - * def unpacks(object packed_bytes): - * """Unpack packed_bytes to object. Returns unpacked object.""" - * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< - * cdef template_context ctx - * cdef size_t off = 0 - */ - __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_p = __pyx_t_1; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 - * cdef const_char_ptr p = packed_bytes - * cdef template_context ctx - * cdef size_t off = 0 # <<<<<<<<<<<<<< - * template_init(&ctx) - * template_execute(&ctx, p, len(packed_bytes), &off) - */ - __pyx_v_off = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 - * cdef template_context ctx - * cdef size_t off = 0 - * template_init(&ctx) # <<<<<<<<<<<<<< - * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) - */ - template_init((&__pyx_v_ctx)); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 - * cdef size_t off = 0 - * template_init(&ctx) - * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< - * return template_data(&ctx) - * - */ - __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 - * template_init(&ctx) - * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) # <<<<<<<<<<<<<< - * - * def unpack(object stream): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_r = __pyx_t_3; - __pyx_t_3 = 0; - goto __pyx_L0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.unpacks"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 - * return template_data(&ctx) - * - * def unpack(object stream): # <<<<<<<<<<<<<< - * """unpack from stream.""" - * packed = stream.read() - */ - -static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream); /*proto*/ -static char __pyx_doc_7msgpack_unpack[] = "unpack from stream."; -static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream) { - PyObject *__pyx_v_packed; - PyObject *__pyx_r = NULL; - PyObject *__pyx_1 = 0; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - __Pyx_SetupRefcountContext("unpack"); - __pyx_self = __pyx_self; - __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 - * def unpack(object stream): - * """unpack from stream.""" - * packed = stream.read() # <<<<<<<<<<<<<< - * return unpacks(packed) - * - */ - __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_v_packed); - __pyx_v_packed = __pyx_t_2; - __pyx_t_2 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 - * """unpack from stream.""" - * packed = stream.read() - * return unpacks(packed) # <<<<<<<<<<<<<< - * - * cdef class Unpacker: - */ - __Pyx_XDECREF(__pyx_r); - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_packed); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_packed); - __Pyx_GIVEREF(__pyx_v_packed); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("msgpack.unpack"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_packed); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} -static struct __pyx_vtabstruct_7msgpack_Packer __pyx_vtable_7msgpack_Packer; - -static PyObject *__pyx_tp_new_7msgpack_Packer(PyTypeObject *t, PyObject *a, PyObject *k) { - struct __pyx_obj_7msgpack_Packer *p; - PyObject *o = (*t->tp_alloc)(t, 0); - if (!o) return 0; - p = ((struct __pyx_obj_7msgpack_Packer *)o); - p->__pyx_vtab = __pyx_vtabptr_7msgpack_Packer; - p->strm = Py_None; Py_INCREF(Py_None); - return o; -} - -static void __pyx_tp_dealloc_7msgpack_Packer(PyObject *o) { - struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; - Py_XDECREF(p->strm); - (*Py_TYPE(o)->tp_free)(o); -} - -static int __pyx_tp_traverse_7msgpack_Packer(PyObject *o, visitproc v, void *a) { - int e; - struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; - if (p->strm) { - e = (*v)(p->strm, a); if (e) return e; - } - return 0; -} - -static int __pyx_tp_clear_7msgpack_Packer(PyObject *o) { - struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; - PyObject* tmp; - tmp = ((PyObject*)p->strm); - p->strm = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - return 0; -} - -static struct PyMethodDef __pyx_methods_7msgpack_Packer[] = { - {__Pyx_NAMESTR("__del__"), (PyCFunction)__pyx_pf_7msgpack_6Packer___del__, METH_NOARGS, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("flush"), (PyCFunction)__pyx_pf_7msgpack_6Packer_flush, METH_NOARGS, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_flush)}, - {__Pyx_NAMESTR("pack_list"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_list, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_list)}, - {__Pyx_NAMESTR("pack_dict"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_dict, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_dict)}, - {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, - {0, 0, 0, 0} -}; - -static PyNumberMethods __pyx_tp_as_number_Packer = { - 0, /*nb_add*/ - 0, /*nb_subtract*/ - 0, /*nb_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_divide*/ - #endif - 0, /*nb_remainder*/ - 0, /*nb_divmod*/ - 0, /*nb_power*/ - 0, /*nb_negative*/ - 0, /*nb_positive*/ - 0, /*nb_absolute*/ - 0, /*nb_nonzero*/ - 0, /*nb_invert*/ - 0, /*nb_lshift*/ - 0, /*nb_rshift*/ - 0, /*nb_and*/ - 0, /*nb_xor*/ - 0, /*nb_or*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_coerce*/ - #endif - 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else - 0, /*nb_long*/ - #endif - 0, /*nb_float*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_oct*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*nb_hex*/ - #endif - 0, /*nb_inplace_add*/ - 0, /*nb_inplace_subtract*/ - 0, /*nb_inplace_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_inplace_divide*/ - #endif - 0, /*nb_inplace_remainder*/ - 0, /*nb_inplace_power*/ - 0, /*nb_inplace_lshift*/ - 0, /*nb_inplace_rshift*/ - 0, /*nb_inplace_and*/ - 0, /*nb_inplace_xor*/ - 0, /*nb_inplace_or*/ - 0, /*nb_floor_divide*/ - 0, /*nb_true_divide*/ - 0, /*nb_inplace_floor_divide*/ - 0, /*nb_inplace_true_divide*/ - #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) - 0, /*nb_index*/ - #endif -}; - -static PySequenceMethods __pyx_tp_as_sequence_Packer = { - 0, /*sq_length*/ - 0, /*sq_concat*/ - 0, /*sq_repeat*/ - 0, /*sq_item*/ - 0, /*sq_slice*/ - 0, /*sq_ass_item*/ - 0, /*sq_ass_slice*/ - 0, /*sq_contains*/ - 0, /*sq_inplace_concat*/ - 0, /*sq_inplace_repeat*/ -}; - -static PyMappingMethods __pyx_tp_as_mapping_Packer = { - 0, /*mp_length*/ - 0, /*mp_subscript*/ - 0, /*mp_ass_subscript*/ -}; - -static PyBufferProcs __pyx_tp_as_buffer_Packer = { - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getreadbuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getwritebuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getsegcount*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getcharbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_getbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_releasebuffer*/ - #endif -}; - -PyTypeObject __pyx_type_7msgpack_Packer = { - PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("msgpack.Packer"), /*tp_name*/ - sizeof(struct __pyx_obj_7msgpack_Packer), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_7msgpack_Packer, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - &__pyx_tp_as_number_Packer, /*tp_as_number*/ - &__pyx_tp_as_sequence_Packer, /*tp_as_sequence*/ - &__pyx_tp_as_mapping_Packer, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - &__pyx_tp_as_buffer_Packer, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - __Pyx_DOCSTR("Packer that pack data into strm.\n\n strm must have `write(bytes)` method.\n size specifies local buffer size.\n "), /*tp_doc*/ - __pyx_tp_traverse_7msgpack_Packer, /*tp_traverse*/ - __pyx_tp_clear_7msgpack_Packer, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_7msgpack_Packer, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - __pyx_pf_7msgpack_6Packer___init__, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_7msgpack_Packer, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ -}; - -static PyObject *__pyx_tp_new_7msgpack_Unpacker(PyTypeObject *t, PyObject *a, PyObject *k) { - PyObject *o = (*t->tp_alloc)(t, 0); - if (!o) return 0; - return o; -} - -static void __pyx_tp_dealloc_7msgpack_Unpacker(PyObject *o) { - (*Py_TYPE(o)->tp_free)(o); -} - -static struct PyMethodDef __pyx_methods_7msgpack_Unpacker[] = { - {0, 0, 0, 0} -}; - -static PyNumberMethods __pyx_tp_as_number_Unpacker = { - 0, /*nb_add*/ - 0, /*nb_subtract*/ - 0, /*nb_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_divide*/ - #endif - 0, /*nb_remainder*/ - 0, /*nb_divmod*/ - 0, /*nb_power*/ - 0, /*nb_negative*/ - 0, /*nb_positive*/ - 0, /*nb_absolute*/ - 0, /*nb_nonzero*/ - 0, /*nb_invert*/ - 0, /*nb_lshift*/ - 0, /*nb_rshift*/ - 0, /*nb_and*/ - 0, /*nb_xor*/ - 0, /*nb_or*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_coerce*/ - #endif - 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else - 0, /*nb_long*/ - #endif - 0, /*nb_float*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_oct*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*nb_hex*/ - #endif - 0, /*nb_inplace_add*/ - 0, /*nb_inplace_subtract*/ - 0, /*nb_inplace_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_inplace_divide*/ - #endif - 0, /*nb_inplace_remainder*/ - 0, /*nb_inplace_power*/ - 0, /*nb_inplace_lshift*/ - 0, /*nb_inplace_rshift*/ - 0, /*nb_inplace_and*/ - 0, /*nb_inplace_xor*/ - 0, /*nb_inplace_or*/ - 0, /*nb_floor_divide*/ - 0, /*nb_true_divide*/ - 0, /*nb_inplace_floor_divide*/ - 0, /*nb_inplace_true_divide*/ - #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) - 0, /*nb_index*/ - #endif -}; - -static PySequenceMethods __pyx_tp_as_sequence_Unpacker = { - 0, /*sq_length*/ - 0, /*sq_concat*/ - 0, /*sq_repeat*/ - 0, /*sq_item*/ - 0, /*sq_slice*/ - 0, /*sq_ass_item*/ - 0, /*sq_ass_slice*/ - 0, /*sq_contains*/ - 0, /*sq_inplace_concat*/ - 0, /*sq_inplace_repeat*/ -}; - -static PyMappingMethods __pyx_tp_as_mapping_Unpacker = { - 0, /*mp_length*/ - 0, /*mp_subscript*/ - 0, /*mp_ass_subscript*/ -}; - -static PyBufferProcs __pyx_tp_as_buffer_Unpacker = { - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getreadbuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getwritebuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getsegcount*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getcharbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_getbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_releasebuffer*/ - #endif -}; - -PyTypeObject __pyx_type_7msgpack_Unpacker = { - PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("msgpack.Unpacker"), /*tp_name*/ - sizeof(struct __pyx_obj_7msgpack_Unpacker), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_7msgpack_Unpacker, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - &__pyx_tp_as_number_Unpacker, /*tp_as_number*/ - &__pyx_tp_as_sequence_Unpacker, /*tp_as_sequence*/ - &__pyx_tp_as_mapping_Unpacker, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - &__pyx_tp_as_buffer_Unpacker, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ - __Pyx_DOCSTR("Do nothing. This function is for symmetric to Packer"), /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_7msgpack_Unpacker, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_7msgpack_Unpacker, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ -}; - -static struct PyMethodDef __pyx_methods[] = { - {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("packs"), (PyCFunction)__pyx_pf_7msgpack_packs, METH_O, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("unpacks"), (PyCFunction)__pyx_pf_7msgpack_unpacks, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpacks)}, - {__Pyx_NAMESTR("unpack"), (PyCFunction)__pyx_pf_7msgpack_unpack, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpack)}, - {0, 0, 0, 0} -}; - -static void __pyx_init_filenames(void); /*proto*/ - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef __pyx_moduledef = { - PyModuleDef_HEAD_INIT, - __Pyx_NAMESTR("msgpack"), - 0, /* m_doc */ - -1, /* m_size */ - __pyx_methods /* m_methods */, - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ -}; -#endif - -static __Pyx_StringTabEntry __pyx_string_tab[] = { - {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, - {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, - {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, - {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, - {&__pyx_kp_pack_list, __pyx_k_pack_list, sizeof(__pyx_k_pack_list), 1, 1, 1}, - {&__pyx_kp_pack_dict, __pyx_k_pack_dict, sizeof(__pyx_k_pack_dict), 1, 1, 1}, - {&__pyx_kp_pack, __pyx_k_pack, sizeof(__pyx_k_pack), 1, 1, 1}, - {&__pyx_kp_unpack, __pyx_k_unpack, sizeof(__pyx_k_unpack), 1, 1, 1}, - {&__pyx_kp_strm, __pyx_k_strm, sizeof(__pyx_k_strm), 1, 1, 1}, - {&__pyx_kp_size, __pyx_k_size, sizeof(__pyx_k_size), 1, 1, 1}, - {&__pyx_kp_len, __pyx_k_len, sizeof(__pyx_k_len), 1, 1, 1}, - {&__pyx_kp_obj, __pyx_k_obj, sizeof(__pyx_k_obj), 1, 1, 1}, - {&__pyx_kp_o, __pyx_k_o, sizeof(__pyx_k_o), 1, 1, 1}, - {&__pyx_kp_stream, __pyx_k_stream, sizeof(__pyx_k_stream), 1, 1, 1}, - {&__pyx_kp_packed_bytes, __pyx_k_packed_bytes, sizeof(__pyx_k_packed_bytes), 1, 1, 1}, - {&__pyx_kp_cStringIO, __pyx_k_cStringIO, sizeof(__pyx_k_cStringIO), 1, 1, 1}, - {&__pyx_kp_StringIO, __pyx_k_StringIO, sizeof(__pyx_k_StringIO), 1, 1, 1}, - {&__pyx_kp_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 1, 1, 1}, - {&__pyx_kp_unpacks, __pyx_k_unpacks, sizeof(__pyx_k_unpacks), 1, 1, 1}, - {&__pyx_kp_write, __pyx_k_write, sizeof(__pyx_k_write), 1, 1, 1}, - {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 1, 0}, - {&__pyx_kp_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 1, 1, 1}, - {&__pyx_kp_iteritems, __pyx_k_iteritems, sizeof(__pyx_k_iteritems), 1, 1, 1}, - {&__pyx_kp_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 1, 1, 1}, - {&__pyx_kp_getvalue, __pyx_k_getvalue, sizeof(__pyx_k_getvalue), 1, 1, 1}, - {&__pyx_kp_read, __pyx_k_read, sizeof(__pyx_k_read), 1, 1, 1}, - {&__pyx_kp_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 0}, - {&__pyx_kp_4, __pyx_k_4, sizeof(__pyx_k_4), 0, 0, 0}, - {0, 0, 0, 0, 0, 0} -}; -static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - return 0; - __pyx_L1_error:; - return -1; -} - -static int __Pyx_InitGlobals(void) { - if (__Pyx_InitStrings(__pyx_string_tab) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - return 0; - __pyx_L1_error:; - return -1; -} - -#if PY_MAJOR_VERSION < 3 -PyMODINIT_FUNC initmsgpack(void); /*proto*/ -PyMODINIT_FUNC initmsgpack(void) -#else -PyMODINIT_FUNC PyInit_msgpack(void); /*proto*/ -PyMODINIT_FUNC PyInit_msgpack(void) -#endif -{ - PyObject *__pyx_1 = 0; - PyObject *__pyx_2 = 0; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - #ifdef CYTHON_REFNANNY - void* __pyx_refchk = NULL; - __Pyx_Refnanny = __Pyx_ImportRefcountAPI("refnanny"); - if (!__Pyx_Refnanny) { - PyErr_Clear(); - __Pyx_Refnanny = __Pyx_ImportRefcountAPI("Cython.Runtime.refnanny"); - if (!__Pyx_Refnanny) - Py_FatalError("failed to import refnanny module"); - } - __pyx_refchk = __Pyx_Refnanny->NewContext("PyMODINIT_FUNC PyInit_msgpack(void)", __LINE__, __FILE__); - #endif - __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /*--- Library function declarations ---*/ - __pyx_init_filenames(); - /*--- Threads initialization code ---*/ - #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS - #ifdef WITH_THREAD /* Python build with threading support? */ - PyEval_InitThreads(); - #endif - #endif - /*--- Initialize various global constants etc. ---*/ - if (unlikely(__Pyx_InitGlobals() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /*--- Module creation code ---*/ - #if PY_MAJOR_VERSION < 3 - __pyx_m = Py_InitModule4(__Pyx_NAMESTR("msgpack"), __pyx_methods, 0, 0, PYTHON_API_VERSION); - #else - __pyx_m = PyModule_Create(&__pyx_moduledef); - #endif - if (!__pyx_m) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #if PY_MAJOR_VERSION < 3 - Py_INCREF(__pyx_m); - #endif - __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); - if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - if (__pyx_module_is_main_msgpack) { - if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - } - /*--- Builtin init code ---*/ - if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_skip_dispatch = 0; - /*--- Global init code ---*/ - /*--- Function export code ---*/ - /*--- Type init code ---*/ - __pyx_vtabptr_7msgpack_Packer = &__pyx_vtable_7msgpack_Packer; - #if PY_MAJOR_VERSION >= 3 - __pyx_vtable_7msgpack_Packer.__pack = (PyObject *(*)(struct __pyx_obj_7msgpack_Packer *, PyObject *))__pyx_f_7msgpack_6Packer___pack; - #else - *(void(**)(void))&__pyx_vtable_7msgpack_Packer.__pack = (void(*)(void))__pyx_f_7msgpack_6Packer___pack; - #endif - if (PyType_Ready(&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetVtable(__pyx_type_7msgpack_Packer.tp_dict, __pyx_vtabptr_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetAttrString(__pyx_m, "Packer", (PyObject *)&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_7msgpack_Packer = &__pyx_type_7msgpack_Packer; - if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_7msgpack_Unpacker = &__pyx_type_7msgpack_Unpacker; - /*--- Type import code ---*/ - /*--- Function import code ---*/ - /*--- Execution code ---*/ - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 - * # coding: utf-8 - * - * from cStringIO import StringIO # <<<<<<<<<<<<<< - * - * cdef extern from "Python.h": - */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_kp_StringIO); - PyList_SET_ITEM(__pyx_t_1, 0, __pyx_kp_StringIO); - __Pyx_GIVEREF(__pyx_kp_StringIO); - __pyx_1 = __Pyx_Import(__pyx_kp_cStringIO, ((PyObject *)__pyx_t_1)); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __pyx_2 = PyObject_GetAttr(__pyx_1, __pyx_kp_StringIO); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_2); - if (PyObject_SetAttr(__pyx_m, __pyx_kp_StringIO, __pyx_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_2); __pyx_2 = 0; - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 - * - * - * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< - * - * cdef class Packer: - */ - __pyx_v_7msgpack_BUFF_SIZE = 2048; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 - * raise TypeError, "can't serialize %r" % (o,) - * - * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< - * self.__pack(obj) - * if flush: - */ - __pyx_t_1 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_k_1 = __pyx_t_1; - __pyx_t_1 = 0; - __Pyx_GIVEREF(__pyx_k_1); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 - * cdef class Unpacker: - * """Do nothing. This function is for symmetric to Packer""" - * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< - */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_1); - __Pyx_GIVEREF(__pyx_1); - __pyx_1 = 0; - __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - PyType_Modified(__pyx_ptype_7msgpack_Unpacker); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_2); - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("msgpack"); - Py_DECREF(__pyx_m); __pyx_m = 0; - __pyx_L0:; - __Pyx_FinishRefcountContext(); - #if PY_MAJOR_VERSION < 3 - return; - #else - return __pyx_m; - #endif -} - -static const char *__pyx_filenames[] = { - "msgpack.pyx", -}; - -/* Runtime support code */ - -static void __pyx_init_filenames(void) { - __pyx_f = __pyx_filenames; -} - -static void __Pyx_RaiseDoubleKeywordsError( - const char* func_name, - PyObject* kw_name) -{ - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION >= 3 - "%s() got multiple values for keyword argument '%U'", func_name, kw_name); - #else - "%s() got multiple values for keyword argument '%s'", func_name, - PyString_AS_STRING(kw_name)); - #endif -} - -static void __Pyx_RaiseArgtupleInvalid( - const char* func_name, - int exact, - Py_ssize_t num_min, - Py_ssize_t num_max, - Py_ssize_t num_found) -{ - Py_ssize_t num_expected; - const char *number, *more_or_less; - - if (num_found < num_min) { - num_expected = num_min; - more_or_less = "at least"; - } else { - num_expected = num_max; - more_or_less = "at most"; - } - if (exact) { - more_or_less = "exactly"; - } - number = (num_expected == 1) ? "" : "s"; - PyErr_Format(PyExc_TypeError, - #if PY_VERSION_HEX < 0x02050000 - "%s() takes %s %d positional argument%s (%d given)", - #else - "%s() takes %s %zd positional argument%s (%zd given)", - #endif - func_name, more_or_less, num_expected, number, num_found); -} - -static int __Pyx_ParseOptionalKeywords( - PyObject *kwds, - PyObject **argnames[], - PyObject *kwds2, - PyObject *values[], - Py_ssize_t num_pos_args, - const char* function_name) -{ - PyObject *key = 0, *value = 0; - Py_ssize_t pos = 0; - PyObject*** name; - PyObject*** first_kw_arg = argnames + num_pos_args; - - while (PyDict_Next(kwds, &pos, &key, &value)) { - name = first_kw_arg; - while (*name && (**name != key)) name++; - if (*name) { - values[name-argnames] = value; - } else { - #if PY_MAJOR_VERSION < 3 - if (unlikely(!PyString_CheckExact(key)) && unlikely(!PyString_Check(key))) { - #else - if (unlikely(!PyUnicode_CheckExact(key)) && unlikely(!PyUnicode_Check(key))) { - #endif - goto invalid_keyword_type; - } else { - for (name = first_kw_arg; *name; name++) { - #if PY_MAJOR_VERSION >= 3 - if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && - PyUnicode_Compare(**name, key) == 0) break; - #else - if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && - _PyString_Eq(**name, key)) break; - #endif - } - if (*name) { - values[name-argnames] = value; - } else { - /* unexpected keyword found */ - for (name=argnames; name != first_kw_arg; name++) { - if (**name == key) goto arg_passed_twice; - #if PY_MAJOR_VERSION >= 3 - if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && - PyUnicode_Compare(**name, key) == 0) goto arg_passed_twice; - #else - if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && - _PyString_Eq(**name, key)) goto arg_passed_twice; - #endif - } - if (kwds2) { - if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; - } else { - goto invalid_keyword; - } - } - } - } - } - return 0; -arg_passed_twice: - __Pyx_RaiseDoubleKeywordsError(function_name, **name); - goto bad; -invalid_keyword_type: - PyErr_Format(PyExc_TypeError, - "%s() keywords must be strings", function_name); - goto bad; -invalid_keyword: - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION < 3 - "%s() got an unexpected keyword argument '%s'", - function_name, PyString_AsString(key)); - #else - "%s() got an unexpected keyword argument '%U'", - function_name, key); - #endif -bad: - return -1; -} - -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list) { - PyObject *__import__ = 0; - PyObject *empty_list = 0; - PyObject *module = 0; - PyObject *global_dict = 0; - PyObject *empty_dict = 0; - PyObject *list; - __import__ = __Pyx_GetAttrString(__pyx_b, "__import__"); - if (!__import__) - goto bad; - if (from_list) - list = from_list; - else { - empty_list = PyList_New(0); - if (!empty_list) - goto bad; - list = empty_list; - } - global_dict = PyModule_GetDict(__pyx_m); - if (!global_dict) - goto bad; - empty_dict = PyDict_New(); - if (!empty_dict) - goto bad; - module = PyObject_CallFunctionObjArgs(__import__, - name, global_dict, empty_dict, list, NULL); -bad: - Py_XDECREF(empty_list); - Py_XDECREF(__import__); - Py_XDECREF(empty_dict); - return module; -} - -static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name) { - PyObject *result; - result = PyObject_GetAttr(dict, name); - if (!result) - PyErr_SetObject(PyExc_NameError, name); - return result; -} - -static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { - PyErr_Format(PyExc_ValueError, - #if PY_VERSION_HEX < 0x02050000 - "need more than %d value%s to unpack", (int)index, - #else - "need more than %zd value%s to unpack", index, - #endif - (index == 1) ? "" : "s"); -} - -static INLINE void __Pyx_RaiseTooManyValuesError(void) { - PyErr_SetString(PyExc_ValueError, "too many values to unpack"); -} - -static PyObject *__Pyx_UnpackItem(PyObject *iter, Py_ssize_t index) { - PyObject *item; - if (!(item = PyIter_Next(iter))) { - if (!PyErr_Occurred()) { - __Pyx_RaiseNeedMoreValuesError(index); - } - } - return item; -} - -static int __Pyx_EndUnpack(PyObject *iter) { - PyObject *item; - if ((item = PyIter_Next(iter))) { - Py_DECREF(item); - __Pyx_RaiseTooManyValuesError(); - return -1; - } - else if (!PyErr_Occurred()) - return 0; - else - return -1; -} - -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb) { - Py_XINCREF(type); - Py_XINCREF(value); - Py_XINCREF(tb); - /* First, check the traceback argument, replacing None with NULL. */ - if (tb == Py_None) { - Py_DECREF(tb); - tb = 0; - } - else if (tb != NULL && !PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto raise_error; - } - /* Next, replace a missing value with None */ - if (value == NULL) { - value = Py_None; - Py_INCREF(value); - } - #if PY_VERSION_HEX < 0x02050000 - if (!PyClass_Check(type)) - #else - if (!PyType_Check(type)) - #endif - { - /* Raising an instance. The value should be a dummy. */ - if (value != Py_None) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto raise_error; - } - /* Normalize to raise , */ - Py_DECREF(value); - value = type; - #if PY_VERSION_HEX < 0x02050000 - if (PyInstance_Check(type)) { - type = (PyObject*) ((PyInstanceObject*)type)->in_class; - Py_INCREF(type); - } - else { - type = 0; - PyErr_SetString(PyExc_TypeError, - "raise: exception must be an old-style class or instance"); - goto raise_error; - } - #else - type = (PyObject*) Py_TYPE(type); - Py_INCREF(type); - if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto raise_error; - } - #endif - } - __Pyx_ErrRestore(type, value, tb); - return; -raise_error: - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(tb); - return; -} - -static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyThreadState *tstate = PyThreadState_GET(); - -#if PY_MAJOR_VERSION >= 3 - /* Note: this is a temporary work-around to prevent crashes in Python 3.0 */ - if ((tstate->exc_type != NULL) & (tstate->exc_type != Py_None)) { - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - PyErr_NormalizeException(&type, &value, &tb); - PyErr_NormalizeException(&tmp_type, &tmp_value, &tmp_tb); - tstate->exc_type = 0; - tstate->exc_value = 0; - tstate->exc_traceback = 0; - PyException_SetContext(value, tmp_value); - Py_DECREF(tmp_type); - Py_XDECREF(tmp_tb); - } -#endif - - tmp_type = tstate->curexc_type; - tmp_value = tstate->curexc_value; - tmp_tb = tstate->curexc_traceback; - tstate->curexc_type = type; - tstate->curexc_value = value; - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -} - -static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb) { - PyThreadState *tstate = PyThreadState_GET(); - *type = tstate->curexc_type; - *value = tstate->curexc_value; - *tb = tstate->curexc_traceback; - - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; -} - - -static INLINE int __Pyx_StrEq(const char *s1, const char *s2) { - while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } - return *s1 == *s2; -} - -static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject* x) { - if (sizeof(unsigned char) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(unsigned char)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (unsigned char)-1; - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned char"); - return (unsigned char)-1; - } - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to unsigned char"); - return (unsigned char)-1; - } - return (unsigned char)val; - } - return (unsigned char)__Pyx_PyInt_AsUnsignedLong(x); -} - -static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject* x) { - if (sizeof(unsigned short) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(unsigned short)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (unsigned short)-1; - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned short"); - return (unsigned short)-1; - } - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to unsigned short"); - return (unsigned short)-1; - } - return (unsigned short)val; - } - return (unsigned short)__Pyx_PyInt_AsUnsignedLong(x); -} - -static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject* x) { - if (sizeof(unsigned int) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(unsigned int)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (unsigned int)-1; - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned int"); - return (unsigned int)-1; - } - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to unsigned int"); - return (unsigned int)-1; - } - return (unsigned int)val; - } - return (unsigned int)__Pyx_PyInt_AsUnsignedLong(x); -} - -static INLINE char __Pyx_PyInt_AsChar(PyObject* x) { - if (sizeof(char) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(char)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (char)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to char"); - return (char)-1; - } - return (char)val; - } - return (char)__Pyx_PyInt_AsLong(x); -} - -static INLINE short __Pyx_PyInt_AsShort(PyObject* x) { - if (sizeof(short) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(short)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (short)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to short"); - return (short)-1; - } - return (short)val; - } - return (short)__Pyx_PyInt_AsLong(x); -} - -static INLINE int __Pyx_PyInt_AsInt(PyObject* x) { - if (sizeof(int) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(int)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (int)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to int"); - return (int)-1; - } - return (int)val; - } - return (int)__Pyx_PyInt_AsLong(x); -} - -static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject* x) { - if (sizeof(signed char) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(signed char)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (signed char)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to signed char"); - return (signed char)-1; - } - return (signed char)val; - } - return (signed char)__Pyx_PyInt_AsSignedLong(x); -} - -static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject* x) { - if (sizeof(signed short) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(signed short)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (signed short)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to signed short"); - return (signed short)-1; - } - return (signed short)val; - } - return (signed short)__Pyx_PyInt_AsSignedLong(x); -} - -static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject* x) { - if (sizeof(signed int) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(signed int)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (signed int)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to signed int"); - return (signed int)-1; - } - return (signed int)val; - } - return (signed int)__Pyx_PyInt_AsSignedLong(x); -} - -static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned long"); - return (unsigned long)-1; - } - return (unsigned long)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - if (unlikely(Py_SIZE(x) < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned long"); - return (unsigned long)-1; - } - return PyLong_AsUnsignedLong(x); - } else { - unsigned long val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (unsigned long)-1; - val = __Pyx_PyInt_AsUnsignedLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned PY_LONG_LONG"); - return (unsigned PY_LONG_LONG)-1; - } - return (unsigned PY_LONG_LONG)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - if (unlikely(Py_SIZE(x) < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned PY_LONG_LONG"); - return (unsigned PY_LONG_LONG)-1; - } - return PyLong_AsUnsignedLongLong(x); - } else { - unsigned PY_LONG_LONG val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (unsigned PY_LONG_LONG)-1; - val = __Pyx_PyInt_AsUnsignedLongLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE long __Pyx_PyInt_AsLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (long)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLong(x); - } else { - long val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (long)-1; - val = __Pyx_PyInt_AsLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (PY_LONG_LONG)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLongLong(x); - } else { - PY_LONG_LONG val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (PY_LONG_LONG)-1; - val = __Pyx_PyInt_AsLongLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (signed long)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLong(x); - } else { - signed long val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (signed long)-1; - val = __Pyx_PyInt_AsSignedLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (signed PY_LONG_LONG)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLongLong(x); - } else { - signed PY_LONG_LONG val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (signed PY_LONG_LONG)-1; - val = __Pyx_PyInt_AsSignedLongLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static void __Pyx_WriteUnraisable(const char *name) { - PyObject *old_exc, *old_val, *old_tb; - PyObject *ctx; - __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); - #if PY_MAJOR_VERSION < 3 - ctx = PyString_FromString(name); - #else - ctx = PyUnicode_FromString(name); - #endif - __Pyx_ErrRestore(old_exc, old_val, old_tb); - if (!ctx) { - PyErr_WriteUnraisable(Py_None); - } else { - PyErr_WriteUnraisable(ctx); - Py_DECREF(ctx); - } -} - -static int __Pyx_SetVtable(PyObject *dict, void *vtable) { - PyObject *pycobj = 0; - int result; - - pycobj = PyCObject_FromVoidPtr(vtable, 0); - if (!pycobj) - goto bad; - if (PyDict_SetItemString(dict, "__pyx_vtable__", pycobj) < 0) - goto bad; - result = 0; - goto done; - -bad: - result = -1; -done: - Py_XDECREF(pycobj); - return result; -} - -#include "compile.h" -#include "frameobject.h" -#include "traceback.h" - -static void __Pyx_AddTraceback(const char *funcname) { - PyObject *py_srcfile = 0; - PyObject *py_funcname = 0; - PyObject *py_globals = 0; - PyObject *empty_string = 0; - PyCodeObject *py_code = 0; - PyFrameObject *py_frame = 0; - - #if PY_MAJOR_VERSION < 3 - py_srcfile = PyString_FromString(__pyx_filename); - #else - py_srcfile = PyUnicode_FromString(__pyx_filename); - #endif - if (!py_srcfile) goto bad; - if (__pyx_clineno) { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); - #else - py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); - #endif - } - else { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromString(funcname); - #else - py_funcname = PyUnicode_FromString(funcname); - #endif - } - if (!py_funcname) goto bad; - py_globals = PyModule_GetDict(__pyx_m); - if (!py_globals) goto bad; - #if PY_MAJOR_VERSION < 3 - empty_string = PyString_FromStringAndSize("", 0); - #else - empty_string = PyBytes_FromStringAndSize("", 0); - #endif - if (!empty_string) goto bad; - py_code = PyCode_New( - 0, /*int argcount,*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*int kwonlyargcount,*/ - #endif - 0, /*int nlocals,*/ - 0, /*int stacksize,*/ - 0, /*int flags,*/ - empty_string, /*PyObject *code,*/ - __pyx_empty_tuple, /*PyObject *consts,*/ - __pyx_empty_tuple, /*PyObject *names,*/ - __pyx_empty_tuple, /*PyObject *varnames,*/ - __pyx_empty_tuple, /*PyObject *freevars,*/ - __pyx_empty_tuple, /*PyObject *cellvars,*/ - py_srcfile, /*PyObject *filename,*/ - py_funcname, /*PyObject *name,*/ - __pyx_lineno, /*int firstlineno,*/ - empty_string /*PyObject *lnotab*/ - ); - if (!py_code) goto bad; - py_frame = PyFrame_New( - PyThreadState_GET(), /*PyThreadState *tstate,*/ - py_code, /*PyCodeObject *code,*/ - py_globals, /*PyObject *globals,*/ - 0 /*PyObject *locals*/ - ); - if (!py_frame) goto bad; - py_frame->f_lineno = __pyx_lineno; - PyTraceBack_Here(py_frame); -bad: - Py_XDECREF(py_srcfile); - Py_XDECREF(py_funcname); - Py_XDECREF(empty_string); - Py_XDECREF(py_code); - Py_XDECREF(py_frame); -} - -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { - while (t->p) { - #if PY_MAJOR_VERSION < 3 - if (t->is_unicode && (!t->is_identifier)) { - *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); - } else if (t->intern) { - *t->p = PyString_InternFromString(t->s); - } else { - *t->p = PyString_FromStringAndSize(t->s, t->n - 1); - } - #else /* Python 3+ has unicode identifiers */ - if (t->is_identifier || (t->is_unicode && t->intern)) { - *t->p = PyUnicode_InternFromString(t->s); - } else if (t->is_unicode) { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } else { - *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); - } - #endif - if (!*t->p) - return -1; - ++t; - } - return 0; -} - -/* Type Conversion Functions */ - -static INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { - if (x == Py_True) return 1; - else if ((x == Py_False) | (x == Py_None)) return 0; - else return PyObject_IsTrue(x); -} - -static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { - PyNumberMethods *m; - const char *name = NULL; - PyObject *res = NULL; -#if PY_VERSION_HEX < 0x03000000 - if (PyInt_Check(x) || PyLong_Check(x)) -#else - if (PyLong_Check(x)) -#endif - return Py_INCREF(x), x; - m = Py_TYPE(x)->tp_as_number; -#if PY_VERSION_HEX < 0x03000000 - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Int(x); - } - else if (m && m->nb_long) { - name = "long"; - res = PyNumber_Long(x); - } -#else - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Long(x); - } -#endif - if (res) { -#if PY_VERSION_HEX < 0x03000000 - if (!PyInt_Check(res) && !PyLong_Check(res)) { -#else - if (!PyLong_Check(res)) { -#endif - PyErr_Format(PyExc_TypeError, - "__%s__ returned non-%s (type %.200s)", - name, name, Py_TYPE(res)->tp_name); - Py_DECREF(res); - return NULL; - } - } - else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, - "an integer is required"); - } - return res; -} - -static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { - Py_ssize_t ival; - PyObject* x = PyNumber_Index(b); - if (!x) return -1; - ival = PyInt_AsSsize_t(x); - Py_DECREF(x); - return ival; -} - -static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { -#if PY_VERSION_HEX < 0x02050000 - if (ival <= LONG_MAX) - return PyInt_FromLong((long)ival); - else { - unsigned char *bytes = (unsigned char *) &ival; - int one = 1; int little = (int)*(unsigned char*)&one; - return _PyLong_FromByteArray(bytes, sizeof(size_t), little, 0); - } -#else - return PyInt_FromSize_t(ival); -#endif -} - -static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject* x) { - unsigned PY_LONG_LONG val = __Pyx_PyInt_AsUnsignedLongLong(x); - if (unlikely(val == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())) { - return (size_t)-1; - } else if (unlikely(val != (unsigned PY_LONG_LONG)(size_t)val)) { - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to size_t"); - return (size_t)-1; - } - return (size_t)val; -} - - diff --git a/msgpack/__init__.py b/msgpack/__init__.py new file mode 100644 index 0000000..797b29c --- /dev/null +++ b/msgpack/__init__.py @@ -0,0 +1,3 @@ +# coding: utf-8 +from _msgpack import * + diff --git a/msgpack.pyx b/msgpack/_msgpack.pyx similarity index 100% rename from msgpack.pyx rename to msgpack/_msgpack.pyx diff --git a/pack.h b/msgpack/pack.h similarity index 100% rename from pack.h rename to msgpack/pack.h diff --git a/unpack.h b/msgpack/unpack.h similarity index 100% rename from unpack.h rename to msgpack/unpack.h diff --git a/setup.py b/setup.py index 65ca412..56b3faa 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,13 @@ from distutils.core import setup, Extension +from Cython.Distutils import build_ext import os version = '0.0.1' PACKAGE_ROOT = os.getcwdu() INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') -msgpack_mod = Extension('msgpack', - sources=['msgpack.c'], +msgpack_mod = Extension('msgpack._msgpack', + sources=['msgpack/_msgpack.pyx'], include_dirs=[INCLUDE_PATH]) desc = 'MessagePack serializer/desirializer.' @@ -28,6 +29,7 @@ setup(name='msgpack', author='Naoki INADA', author_email='songofacandy@gmail.com', version=version, + cmdclass={'build_ext': build_ext}, ext_modules=[msgpack_mod], description=desc, long_description=long_desc, From e814986b4ec017ab124dcf16496a6fefa65a9876 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 13:33:42 +0900 Subject: [PATCH 0155/1172] Fix document miss. --- python/msgpack/_msgpack.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index f24f403..7c07cde 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -79,7 +79,7 @@ cdef class Packer: packer.pack('foo') packer.pack('bar') - This code is same as below code: + This is same to: packer.pack(['foo', 'bar']) """ @@ -94,9 +94,9 @@ cdef class Packer: packer.pack('foo') packer.pack('bar') - This code is same as below code: + This is same to: - packer.pack({'foo', 'bar'}) + packer.pack({'foo': 'bar'}) """ msgpack_pack_map(&self.pk, len) From 9ae7b2b28c567ce0b78a90d9b16375a4568cab1f Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 13:33:42 +0900 Subject: [PATCH 0156/1172] Fix document miss. --- msgpack/_msgpack.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index f24f403..7c07cde 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -79,7 +79,7 @@ cdef class Packer: packer.pack('foo') packer.pack('bar') - This code is same as below code: + This is same to: packer.pack(['foo', 'bar']) """ @@ -94,9 +94,9 @@ cdef class Packer: packer.pack('foo') packer.pack('bar') - This code is same as below code: + This is same to: - packer.pack({'foo', 'bar'}) + packer.pack({'foo': 'bar'}) """ msgpack_pack_map(&self.pk, len) From 6184e17a42c89993e445a668169a76cd5bcad046 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 16 Jun 2009 01:56:04 +0900 Subject: [PATCH 0157/1172] Increase stack size. --- python/unpack.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/unpack.h b/python/unpack.h index e51557f..c98c19a 100644 --- a/python/unpack.h +++ b/python/unpack.h @@ -15,6 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +#define MSGPACK_MAX_STACK_SIZE (1024) #include "msgpack/unpack_define.h" typedef struct { From a90efd70c229b94b81184e94d6b36e727144ff23 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 16 Jun 2009 01:56:04 +0900 Subject: [PATCH 0158/1172] Increase stack size. --- unpack.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unpack.h b/unpack.h index e51557f..c98c19a 100644 --- a/unpack.h +++ b/unpack.h @@ -15,6 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +#define MSGPACK_MAX_STACK_SIZE (1024) #include "msgpack/unpack_define.h" typedef struct { From 4d6e9ffaa2f1626e38fbc7ab5d08578213295a65 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 16 Jun 2009 01:58:07 +0900 Subject: [PATCH 0159/1172] Fix can't pack float values. --- python/msgpack.c | 212 +++++++++++++++++++++++---------------------- python/msgpack.pyx | 2 +- 2 files changed, 108 insertions(+), 106 deletions(-) diff --git a/python/msgpack.c b/python/msgpack.c index 821f65b..7e5c21f 100644 --- a/python/msgpack.c +++ b/python/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Tue Jun 9 13:10:11 2009 */ +/* Generated by Cython 0.11.2 on Tue Jun 16 01:57:05 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -800,13 +800,14 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack int __pyx_t_1; PyObject *__pyx_t_2 = NULL; PY_LONG_LONG __pyx_t_3; - char *__pyx_t_4; - Py_ssize_t __pyx_t_5; - PyObject *__pyx_t_6 = NULL; + double __pyx_t_4; + char *__pyx_t_5; + Py_ssize_t __pyx_t_6; PyObject *__pyx_t_7 = NULL; PyObject *__pyx_t_8 = NULL; - int __pyx_t_9; + PyObject *__pyx_t_9 = NULL; int __pyx_t_10; + int __pyx_t_11; __Pyx_SetupRefcountContext("__pack"); __Pyx_INCREF(__pyx_v_o); __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); @@ -937,7 +938,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< * elif isinstance(o, float): - * fval = 9 + * fval = o */ msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); goto __pyx_L3; @@ -947,7 +948,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< - * fval = 9 + * fval = o * msgpack_pack_double(&self.pk, fval) */ __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); @@ -956,15 +957,16 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): - * fval = 9 # <<<<<<<<<<<<<< + * fval = o # <<<<<<<<<<<<<< * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): */ - __pyx_v_fval = 9; + __pyx_t_4 = __pyx_PyFloat_AsDouble(__pyx_v_o); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_fval = __pyx_t_4; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 * elif isinstance(o, float): - * fval = 9 + * fval = o * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< * elif isinstance(o, str): * rawval = o @@ -974,7 +976,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 - * fval = 9 + * fval = o * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< * rawval = o @@ -990,8 +992,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; + __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_5; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 * elif isinstance(o, str): @@ -1000,8 +1002,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 * rawval = o @@ -1010,8 +1012,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * elif isinstance(o, unicode): * o = o.encode('utf-8') */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_6); goto __pyx_L3; } @@ -1034,18 +1036,18 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack */ __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_6)); + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_7)); __Pyx_INCREF(__pyx_kp_3); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_3); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_kp_3); __Pyx_GIVEREF(__pyx_kp_3); - __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_7), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_v_o); - __pyx_v_o = __pyx_t_7; - __pyx_t_7 = 0; + __pyx_v_o = __pyx_t_8; + __pyx_t_8 = 0; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 * elif isinstance(o, unicode): @@ -1054,8 +1056,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; + __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_5; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 * o = o.encode('utf-8') @@ -1064,8 +1066,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 * rawval = o @@ -1074,8 +1076,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_6); goto __pyx_L3; } @@ -1096,8 +1098,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * for k,v in o.iteritems(): * self.pack(k) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 * elif isinstance(o, dict): @@ -1106,38 +1108,38 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * self.pack(k) * self.pack(v) */ - __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_7 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); - __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (PyList_CheckExact(__pyx_t_7) || PyTuple_CheckExact(__pyx_t_7)) { + __pyx_t_6 = 0; __pyx_t_8 = __pyx_t_7; __Pyx_INCREF(__pyx_t_8); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = -1; __pyx_t_8 = PyObject_GetIter(__pyx_t_7); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); } - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + if (likely(PyList_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyList_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; + } else if (likely(PyTuple_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { + __pyx_t_7 = PyIter_Next(__pyx_t_8); + if (!__pyx_t_7) { if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } - __Pyx_GOTREF(__pyx_t_6); + __Pyx_GOTREF(__pyx_t_7); } - if (PyTuple_CheckExact(__pyx_t_6) && likely(PyTuple_GET_SIZE(__pyx_t_6) == 2)) { - PyObject* tuple = __pyx_t_6; + if (PyTuple_CheckExact(__pyx_t_7) && likely(PyTuple_GET_SIZE(__pyx_t_7) == 2)) { + PyObject* tuple = __pyx_t_7; __pyx_2 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_2); __pyx_3 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_3); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_v_k); __pyx_v_k = __pyx_2; __pyx_2 = 0; @@ -1145,9 +1147,9 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_v = __pyx_3; __pyx_3 = 0; } else { - __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = PyObject_GetIter(__pyx_t_7); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_2); __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} @@ -1169,18 +1171,18 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_k); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); __Pyx_GIVEREF(__pyx_v_k); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_9 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 * for k,v in o.iteritems(): @@ -1189,20 +1191,20 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) */ - __pyx_t_8 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_7 = PyObject_Call(__pyx_t_9, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; goto __pyx_L3; } @@ -1215,12 +1217,12 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack */ __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyTuple_Type))); if (!__pyx_t_1) { - __pyx_t_9 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); - __pyx_t_10 = __pyx_t_9; + __pyx_t_10 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); + __pyx_t_11 = __pyx_t_10; } else { - __pyx_t_10 = __pyx_t_1; + __pyx_t_11 = __pyx_t_1; } - if (__pyx_t_10) { + if (__pyx_t_11) { /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 * self.pack(v) @@ -1229,8 +1231,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * for v in o: * self.pack(v) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): @@ -1240,29 +1242,29 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * else: */ if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); + __pyx_t_6 = 0; __pyx_t_8 = __pyx_v_o; __Pyx_INCREF(__pyx_t_8); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = -1; __pyx_t_8 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); } for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + if (likely(PyList_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyList_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; + } else if (likely(PyTuple_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { + __pyx_t_7 = PyIter_Next(__pyx_t_8); + if (!__pyx_t_7) { if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } - __Pyx_GOTREF(__pyx_t_6); + __Pyx_GOTREF(__pyx_t_7); } __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_t_6; - __pyx_t_6 = 0; + __pyx_v_v = __pyx_t_7; + __pyx_t_7 = 0; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) @@ -1271,20 +1273,20 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * else: * # TODO: Serialize with defalt() like simplejson. */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_9 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; goto __pyx_L3; } /*else*/ { @@ -1296,16 +1298,16 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * * def pack(self, obj, flush=True): */ - __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_7)); + __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_8)); __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_8 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; - __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_9 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_8)); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(((PyObject *)__pyx_t_8)); __pyx_t_8 = 0; + __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_9, 0); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } __pyx_L3:; @@ -1317,9 +1319,9 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_XDECREF(__pyx_2); __Pyx_XDECREF(__pyx_3); __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_6); __Pyx_XDECREF(__pyx_t_7); __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); __Pyx_AddTraceback("msgpack.Packer.__pack"); __pyx_r = 0; __pyx_L0:; diff --git a/python/msgpack.pyx b/python/msgpack.pyx index f24f403..aa4006b 100644 --- a/python/msgpack.pyx +++ b/python/msgpack.pyx @@ -118,7 +118,7 @@ cdef class Packer: intval = o msgpack_pack_long_long(&self.pk, intval) elif isinstance(o, float): - fval = 9 + fval = o msgpack_pack_double(&self.pk, fval) elif isinstance(o, str): rawval = o From 4ab64bf9497eccb15081388a667b01c618878d58 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 16 Jun 2009 01:58:07 +0900 Subject: [PATCH 0160/1172] Fix can't pack float values. --- msgpack.c | 212 ++++++++++++++++++++++++++-------------------------- msgpack.pyx | 2 +- 2 files changed, 108 insertions(+), 106 deletions(-) diff --git a/msgpack.c b/msgpack.c index 821f65b..7e5c21f 100644 --- a/msgpack.c +++ b/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Tue Jun 9 13:10:11 2009 */ +/* Generated by Cython 0.11.2 on Tue Jun 16 01:57:05 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -800,13 +800,14 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack int __pyx_t_1; PyObject *__pyx_t_2 = NULL; PY_LONG_LONG __pyx_t_3; - char *__pyx_t_4; - Py_ssize_t __pyx_t_5; - PyObject *__pyx_t_6 = NULL; + double __pyx_t_4; + char *__pyx_t_5; + Py_ssize_t __pyx_t_6; PyObject *__pyx_t_7 = NULL; PyObject *__pyx_t_8 = NULL; - int __pyx_t_9; + PyObject *__pyx_t_9 = NULL; int __pyx_t_10; + int __pyx_t_11; __Pyx_SetupRefcountContext("__pack"); __Pyx_INCREF(__pyx_v_o); __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); @@ -937,7 +938,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< * elif isinstance(o, float): - * fval = 9 + * fval = o */ msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); goto __pyx_L3; @@ -947,7 +948,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< - * fval = 9 + * fval = o * msgpack_pack_double(&self.pk, fval) */ __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); @@ -956,15 +957,16 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): - * fval = 9 # <<<<<<<<<<<<<< + * fval = o # <<<<<<<<<<<<<< * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): */ - __pyx_v_fval = 9; + __pyx_t_4 = __pyx_PyFloat_AsDouble(__pyx_v_o); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_fval = __pyx_t_4; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 * elif isinstance(o, float): - * fval = 9 + * fval = o * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< * elif isinstance(o, str): * rawval = o @@ -974,7 +976,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 - * fval = 9 + * fval = o * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< * rawval = o @@ -990,8 +992,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; + __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_5; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 * elif isinstance(o, str): @@ -1000,8 +1002,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 * rawval = o @@ -1010,8 +1012,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * elif isinstance(o, unicode): * o = o.encode('utf-8') */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_6); goto __pyx_L3; } @@ -1034,18 +1036,18 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack */ __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_6)); + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_7)); __Pyx_INCREF(__pyx_kp_3); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_3); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_kp_3); __Pyx_GIVEREF(__pyx_kp_3); - __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_7), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_v_o); - __pyx_v_o = __pyx_t_7; - __pyx_t_7 = 0; + __pyx_v_o = __pyx_t_8; + __pyx_t_8 = 0; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 * elif isinstance(o, unicode): @@ -1054,8 +1056,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; + __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_5; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 * o = o.encode('utf-8') @@ -1064,8 +1066,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 * rawval = o @@ -1074,8 +1076,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_6); goto __pyx_L3; } @@ -1096,8 +1098,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * for k,v in o.iteritems(): * self.pack(k) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 * elif isinstance(o, dict): @@ -1106,38 +1108,38 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * self.pack(k) * self.pack(v) */ - __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_7 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); - __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (PyList_CheckExact(__pyx_t_7) || PyTuple_CheckExact(__pyx_t_7)) { + __pyx_t_6 = 0; __pyx_t_8 = __pyx_t_7; __Pyx_INCREF(__pyx_t_8); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = -1; __pyx_t_8 = PyObject_GetIter(__pyx_t_7); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); } - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + if (likely(PyList_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyList_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; + } else if (likely(PyTuple_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { + __pyx_t_7 = PyIter_Next(__pyx_t_8); + if (!__pyx_t_7) { if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } - __Pyx_GOTREF(__pyx_t_6); + __Pyx_GOTREF(__pyx_t_7); } - if (PyTuple_CheckExact(__pyx_t_6) && likely(PyTuple_GET_SIZE(__pyx_t_6) == 2)) { - PyObject* tuple = __pyx_t_6; + if (PyTuple_CheckExact(__pyx_t_7) && likely(PyTuple_GET_SIZE(__pyx_t_7) == 2)) { + PyObject* tuple = __pyx_t_7; __pyx_2 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_2); __pyx_3 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_3); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_v_k); __pyx_v_k = __pyx_2; __pyx_2 = 0; @@ -1145,9 +1147,9 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_v = __pyx_3; __pyx_3 = 0; } else { - __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = PyObject_GetIter(__pyx_t_7); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_2); __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} @@ -1169,18 +1171,18 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_k); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); __Pyx_GIVEREF(__pyx_v_k); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_9 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 * for k,v in o.iteritems(): @@ -1189,20 +1191,20 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) */ - __pyx_t_8 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_7 = PyObject_Call(__pyx_t_9, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; goto __pyx_L3; } @@ -1215,12 +1217,12 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack */ __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyTuple_Type))); if (!__pyx_t_1) { - __pyx_t_9 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); - __pyx_t_10 = __pyx_t_9; + __pyx_t_10 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); + __pyx_t_11 = __pyx_t_10; } else { - __pyx_t_10 = __pyx_t_1; + __pyx_t_11 = __pyx_t_1; } - if (__pyx_t_10) { + if (__pyx_t_11) { /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 * self.pack(v) @@ -1229,8 +1231,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * for v in o: * self.pack(v) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): @@ -1240,29 +1242,29 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * else: */ if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); + __pyx_t_6 = 0; __pyx_t_8 = __pyx_v_o; __Pyx_INCREF(__pyx_t_8); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = -1; __pyx_t_8 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); } for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + if (likely(PyList_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyList_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; + } else if (likely(PyTuple_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { + __pyx_t_7 = PyIter_Next(__pyx_t_8); + if (!__pyx_t_7) { if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } - __Pyx_GOTREF(__pyx_t_6); + __Pyx_GOTREF(__pyx_t_7); } __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_t_6; - __pyx_t_6 = 0; + __pyx_v_v = __pyx_t_7; + __pyx_t_7 = 0; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) @@ -1271,20 +1273,20 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * else: * # TODO: Serialize with defalt() like simplejson. */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_9 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; goto __pyx_L3; } /*else*/ { @@ -1296,16 +1298,16 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * * def pack(self, obj, flush=True): */ - __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_7)); + __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_8)); __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_8 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; - __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_9 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_8)); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(((PyObject *)__pyx_t_8)); __pyx_t_8 = 0; + __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_9, 0); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } __pyx_L3:; @@ -1317,9 +1319,9 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_XDECREF(__pyx_2); __Pyx_XDECREF(__pyx_3); __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_6); __Pyx_XDECREF(__pyx_t_7); __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); __Pyx_AddTraceback("msgpack.Packer.__pack"); __pyx_r = 0; __pyx_L0:; diff --git a/msgpack.pyx b/msgpack.pyx index f24f403..aa4006b 100644 --- a/msgpack.pyx +++ b/msgpack.pyx @@ -118,7 +118,7 @@ cdef class Packer: intval = o msgpack_pack_long_long(&self.pk, intval) elif isinstance(o, float): - fval = 9 + fval = o msgpack_pack_double(&self.pk, fval) elif isinstance(o, str): rawval = o From 2475187c7d7dc32d3abf825f97f9d7acaaa58e47 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 17 Jun 2009 13:45:08 +0900 Subject: [PATCH 0161/1172] Fix refcount leak and optimize list initialization. --- python/msgpack.c | 229 ++++++++++++++++++++++------------------------- python/setup.py | 2 +- python/unpack.h | 45 ++++++++-- 3 files changed, 142 insertions(+), 134 deletions(-) diff --git a/python/msgpack.c b/python/msgpack.c index 7e5c21f..ed0ac7e 100644 --- a/python/msgpack.c +++ b/python/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Tue Jun 16 01:57:05 2009 */ +/* Generated by Cython 0.11.1 on Wed Jun 17 13:44:30 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -103,9 +103,6 @@ #ifndef __cdecl #define __cdecl #endif - #ifndef __fastcall - #define __fastcall - #endif #else #define _USE_MATH_DEFINES #endif @@ -136,7 +133,6 @@ #include "string.h" #include "pack.h" #include "unpack.h" -#define __PYX_USE_C99_COMPLEX defined(_Complex_I) #ifdef __GNUC__ @@ -228,7 +224,6 @@ static const char * __pyx_cfilenm= __FILE__; static const char *__pyx_filename; static const char **__pyx_f; - #ifdef CYTHON_REFNANNY typedef struct { void (*INCREF)(void*, PyObject*, int); @@ -325,7 +320,7 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ /* Type declarations */ -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -343,7 +338,7 @@ struct __pyx_obj_7msgpack_Packer { PyObject *strm; }; -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":200 * return unpacks(packed) * * cdef class Unpacker: # <<<<<<<<<<<<<< @@ -356,7 +351,7 @@ struct __pyx_obj_7msgpack_Unpacker { }; -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -375,12 +370,10 @@ static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; static int __pyx_v_7msgpack_BUFF_SIZE; static PyObject *__pyx_k_1 = 0; static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ -#define __Pyx_MODULE_NAME "msgpack" -int __pyx_module_is_main_msgpack = 0; + +const char *__pyx_modulename = "msgpack"; /* Implementation of msgpack */ -static char __pyx_k___main__[] = "__main__"; -static PyObject *__pyx_kp___main__; static char __pyx_k___init__[] = "__init__"; static PyObject *__pyx_kp___init__; static char __pyx_k___del__[] = "__del__"; @@ -438,7 +431,7 @@ static PyObject *__pyx_kp_4; static char __pyx_k_3[] = "utf-8"; static char __pyx_k_4[] = "can't serialize %r"; -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":51 * cdef object strm * * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< @@ -500,7 +493,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return -1; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":52 * * def __init__(self, strm, int size=0): * if size <= 0: # <<<<<<<<<<<<<< @@ -510,7 +503,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __pyx_t_1 = (__pyx_v_size <= 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":53 * def __init__(self, strm, int size=0): * if size <= 0: * size = BUFF_SIZE # <<<<<<<<<<<<<< @@ -522,7 +515,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * } __pyx_L6:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":55 * size = BUFF_SIZE * * self.strm = strm # <<<<<<<<<<<<<< @@ -535,7 +528,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":56 * * self.strm = strm * self.buff = malloc(size) # <<<<<<<<<<<<<< @@ -544,7 +537,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":57 * self.strm = strm * self.buff = malloc(size) * self.allocated = size # <<<<<<<<<<<<<< @@ -553,7 +546,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":58 * self.buff = malloc(size) * self.allocated = size * self.length = 0 # <<<<<<<<<<<<<< @@ -562,7 +555,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":60 * self.length = 0 * * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< @@ -576,7 +569,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":62 * msgpack_packer_init(&self.pk, self, _packer_write) * * def __del__(self): # <<<<<<<<<<<<<< @@ -589,7 +582,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj PyObject *__pyx_r = NULL; __Pyx_SetupRefcountContext("__del__"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":63 * * def __del__(self): * free(self.buff); # <<<<<<<<<<<<<< @@ -604,7 +597,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":65 * free(self.buff); * * def flush(self): # <<<<<<<<<<<<<< @@ -622,7 +615,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("flush"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":67 * def flush(self): * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: # <<<<<<<<<<<<<< @@ -632,7 +625,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":68 * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< @@ -654,7 +647,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":69 * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 # <<<<<<<<<<<<<< @@ -666,7 +659,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec } __pyx_L5:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":70 * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< @@ -676,7 +669,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":71 * self.length = 0 * if hasattr(self.strm, 'flush'): * self.strm.flush() # <<<<<<<<<<<<<< @@ -707,7 +700,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":73 * self.strm.flush() * * def pack_list(self, len): # <<<<<<<<<<<<<< @@ -722,7 +715,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_list"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":86 * packer.pack(['foo', 'bar']) * """ * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< @@ -743,7 +736,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":88 * msgpack_pack_array(&self.pk, len) * * def pack_dict(self, len): # <<<<<<<<<<<<<< @@ -758,7 +751,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_dict"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":101 * packer.pack({'foo', 'bar'}) * """ * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< @@ -779,7 +772,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":103 * msgpack_pack_map(&self.pk, len) * * cdef __pack(self, object o): # <<<<<<<<<<<<<< @@ -813,7 +806,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":108 * cdef char* rawval * * if o is None: # <<<<<<<<<<<<<< @@ -823,7 +816,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = (__pyx_v_o == Py_None); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":109 * * if o is None: * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< @@ -834,7 +827,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":110 * if o is None: * msgpack_pack_nil(&self.pk) * elif o is True: # <<<<<<<<<<<<<< @@ -847,7 +840,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":111 * msgpack_pack_nil(&self.pk) * elif o is True: * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< @@ -858,7 +851,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":112 * elif o is True: * msgpack_pack_true(&self.pk) * elif o is False: # <<<<<<<<<<<<<< @@ -871,7 +864,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":113 * msgpack_pack_true(&self.pk) * elif o is False: * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< @@ -882,7 +875,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":114 * elif o is False: * msgpack_pack_false(&self.pk) * elif isinstance(o, long): # <<<<<<<<<<<<<< @@ -892,7 +885,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":115 * msgpack_pack_false(&self.pk) * elif isinstance(o, long): * intval = o # <<<<<<<<<<<<<< @@ -902,7 +895,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":116 * elif isinstance(o, long): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -913,7 +906,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":117 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): # <<<<<<<<<<<<<< @@ -923,7 +916,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":118 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): * intval = o # <<<<<<<<<<<<<< @@ -933,7 +926,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":119 * elif isinstance(o, int): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -944,7 +937,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":120 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< @@ -954,7 +947,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): * fval = o # <<<<<<<<<<<<<< @@ -964,7 +957,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_4 = __pyx_PyFloat_AsDouble(__pyx_v_o); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_fval = __pyx_t_4; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":122 * elif isinstance(o, float): * fval = o * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< @@ -975,7 +968,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":123 * fval = o * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< @@ -985,7 +978,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":124 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): * rawval = o # <<<<<<<<<<<<<< @@ -995,7 +988,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":125 * elif isinstance(o, str): * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1005,7 +998,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":126 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1017,7 +1010,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":127 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): # <<<<<<<<<<<<<< @@ -1027,7 +1020,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":128 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): * o = o.encode('utf-8') # <<<<<<<<<<<<<< @@ -1049,7 +1042,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_o = __pyx_t_8; __pyx_t_8 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":129 * elif isinstance(o, unicode): * o = o.encode('utf-8') * rawval = o # <<<<<<<<<<<<<< @@ -1059,7 +1052,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":130 * o = o.encode('utf-8') * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1069,7 +1062,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":131 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1081,7 +1074,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":132 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): # <<<<<<<<<<<<<< @@ -1091,7 +1084,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":133 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1101,7 +1094,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":134 * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): # <<<<<<<<<<<<<< @@ -1164,7 +1157,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_3 = 0; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":135 * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): * self.pack(k) # <<<<<<<<<<<<<< @@ -1184,7 +1177,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":136 * for k,v in o.iteritems(): * self.pack(k) * self.pack(v) # <<<<<<<<<<<<<< @@ -1208,7 +1201,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":137 * self.pack(k) * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< @@ -1224,7 +1217,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } if (__pyx_t_11) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":138 * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1234,7 +1227,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) * for v in o: # <<<<<<<<<<<<<< @@ -1266,7 +1259,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_v = __pyx_t_7; __pyx_t_7 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) * for v in o: * self.pack(v) # <<<<<<<<<<<<<< @@ -1291,7 +1284,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } /*else*/ { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":143 * else: * # TODO: Serialize with defalt() like simplejson. * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< @@ -1333,7 +1326,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -1394,7 +1387,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return NULL; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":146 * * def pack(self, obj, flush=True): * self.__pack(obj) # <<<<<<<<<<<<<< @@ -1405,7 +1398,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":147 * def pack(self, obj, flush=True): * self.__pack(obj) * if flush: # <<<<<<<<<<<<<< @@ -1415,7 +1408,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_2) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":148 * self.__pack(obj) * if flush: * self.flush() # <<<<<<<<<<<<<< @@ -1445,7 +1438,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":150 * self.flush() * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< @@ -1461,7 +1454,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("_packer_write"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":151 * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< @@ -1471,7 +1464,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":152 * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: * if packer.length > 0: # <<<<<<<<<<<<<< @@ -1481,7 +1474,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_packer->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":153 * if packer.length + l > packer.allocated: * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< @@ -1506,7 +1499,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L4:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":154 * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: # <<<<<<<<<<<<<< @@ -1516,7 +1509,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_l > 64); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":155 * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< @@ -1538,7 +1531,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":156 * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) * packer.length = 0 # <<<<<<<<<<<<<< @@ -1550,7 +1543,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":158 * packer.length = 0 * else: * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< @@ -1559,7 +1552,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":159 * else: * memcpy(packer.buff, b, l) * packer.length = l # <<<<<<<<<<<<<< @@ -1573,7 +1566,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":161 * packer.length = l * else: * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< @@ -1582,7 +1575,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":162 * else: * memcpy(packer.buff + packer.length, b, l) * packer.length += l # <<<<<<<<<<<<<< @@ -1593,7 +1586,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L3:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":163 * memcpy(packer.buff + packer.length, b, l) * packer.length += l * return 0 # <<<<<<<<<<<<<< @@ -1616,7 +1609,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":165 * return 0 * * def pack(object o, object stream): # <<<<<<<<<<<<<< @@ -1677,7 +1670,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_L4_argument_unpacking_done:; __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":166 * * def pack(object o, object stream): * packer = Packer(stream) # <<<<<<<<<<<<<< @@ -1696,7 +1689,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":167 * def pack(object o, object stream): * packer = Packer(stream) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1716,7 +1709,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":168 * packer = Packer(stream) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1745,7 +1738,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":170 * packer.flush() * * def packs(object o): # <<<<<<<<<<<<<< @@ -1767,7 +1760,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":171 * * def packs(object o): * buf = StringIO() # <<<<<<<<<<<<<< @@ -1783,7 +1776,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = __pyx_t_1; __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":172 * def packs(object o): * buf = StringIO() * packer = Packer(buf) # <<<<<<<<<<<<<< @@ -1802,7 +1795,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":173 * buf = StringIO() * packer = Packer(buf) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1822,7 +1815,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":174 * packer = Packer(buf) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1836,7 +1829,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":175 * packer.pack(o) * packer.flush() * return buf.getvalue() # <<<<<<<<<<<<<< @@ -1870,7 +1863,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":186 * * * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< @@ -1891,7 +1884,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":188 * def unpacks(object packed_bytes): * """Unpack packed_bytes to object. Returns unpacked object.""" * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< @@ -1901,7 +1894,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_p = __pyx_t_1; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":190 * cdef const_char_ptr p = packed_bytes * cdef template_context ctx * cdef size_t off = 0 # <<<<<<<<<<<<<< @@ -1910,7 +1903,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ __pyx_v_off = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":191 * cdef template_context ctx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< @@ -1919,7 +1912,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ template_init((&__pyx_v_ctx)); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":192 * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< @@ -1929,7 +1922,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":193 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) * return template_data(&ctx) # <<<<<<<<<<<<<< @@ -1955,7 +1948,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":195 * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< @@ -1975,7 +1968,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_self = __pyx_self; __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":197 * def unpack(object stream): * """unpack from stream.""" * packed = stream.read() # <<<<<<<<<<<<<< @@ -1991,7 +1984,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_v_packed = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":198 * """unpack from stream.""" * packed = stream.read() * return unpacks(packed) # <<<<<<<<<<<<<< @@ -2097,11 +2090,7 @@ static PyNumberMethods __pyx_tp_as_number_Packer = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else 0, /*nb_long*/ - #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2257,11 +2246,7 @@ static PyNumberMethods __pyx_tp_as_number_Unpacker = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else 0, /*nb_long*/ - #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2404,7 +2389,6 @@ static struct PyModuleDef __pyx_moduledef = { #endif static __Pyx_StringTabEntry __pyx_string_tab[] = { - {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, @@ -2496,9 +2480,6 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - if (__pyx_module_is_main_msgpack) { - if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - } /*--- Builtin init code ---*/ if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_skip_dispatch = 0; @@ -2522,7 +2503,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) /*--- Function import code ---*/ /*--- Execution code ---*/ - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":3 * # coding: utf-8 * * from cStringIO import StringIO # <<<<<<<<<<<<<< @@ -2543,7 +2524,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __Pyx_DECREF(__pyx_2); __pyx_2 = 0; __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":37 * * * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< @@ -2552,7 +2533,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) */ __pyx_v_7msgpack_BUFF_SIZE = 2048; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -2565,7 +2546,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_t_1 = 0; __Pyx_GIVEREF(__pyx_k_1); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":202 * cdef class Unpacker: * """Do nothing. This function is for symmetric to Packer""" * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< @@ -3368,14 +3349,14 @@ static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { return Py_INCREF(x), x; m = Py_TYPE(x)->tp_as_number; #if PY_VERSION_HEX < 0x03000000 - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Int(x); - } - else if (m && m->nb_long) { + if (m && m->nb_long) { name = "long"; res = PyNumber_Long(x); } + else if (m && m->nb_int) { + name = "int"; + res = PyNumber_Int(x); + } #else if (m && m->nb_int) { name = "int"; diff --git a/python/setup.py b/python/setup.py index e5651a0..4bb8693 100644 --- a/python/setup.py +++ b/python/setup.py @@ -2,7 +2,7 @@ from distutils.core import setup, Extension version = '0.0.1' -msgpack_mod = Extension('msgpack', sources=['msgpack.c']) +msgpack_mod = Extension('msgpack', sources=['msgpack.c'], extra_compile_args=["-O3"]) desc = 'MessagePack serializer/desirializer.' long_desc = desc + """ diff --git a/python/unpack.h b/python/unpack.h index c98c19a..3e99123 100644 --- a/python/unpack.h +++ b/python/unpack.h @@ -20,7 +20,8 @@ #include "msgpack/unpack_define.h" typedef struct { - int reserved; + struct {unsigned int size, last} array_stack[MSGPACK_MAX_STACK_SIZE]; + int array_current; } unpack_user; @@ -42,7 +43,10 @@ struct template_context; typedef struct template_context template_context; static inline msgpack_unpack_object template_callback_root(unpack_user* u) -{ return NULL; } +{ + u->array_current = -1; + return NULL; +} static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) { *o = PyInt_FromLong((long)d); return 0; } @@ -52,8 +56,8 @@ static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_u static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { - if (d >= 0x80000000UL) { - *o = PyLong_FromUnsignedLongLong((unsigned long long)d); + if (d > LONG_MAX) { + *o = PyLong_FromUnsignedLong((unsigned long)d); } else { *o = PyInt_FromLong((long)d); } @@ -92,14 +96,32 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - /* TODO: use PyList_New(n). */ - *o = PyList_New(0); + if (n > 0) { + int cur = ++u->array_current; + u->array_stack[cur].size = n; + u->array_stack[cur].last = 0; + *o = PyList_New(n); + } + else { + *o = PyList_New(0); + } return 0; } static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - PyList_Append(*c, o); + int cur = u->array_current; + int n = u->array_stack[cur].size; + int last = u->array_stack[cur].last; + + PyList_SetItem(*c, last, o); + last++; + if (last >= n) { + u->array_current--; + } + else { + u->array_stack[cur].last = last; + } return 0; } @@ -112,13 +134,18 @@ static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { PyDict_SetItem(*c, k, v); - return 0; + Py_DECREF(k); + Py_DECREF(v); + return 0; } static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { *o = PyString_FromStringAndSize(p, l); - return 0; + if (l < 16) { // without foundation + PyString_InternInPlace(o); + } + return 0; } #include "msgpack/unpack_template.h" From 8ca2c441c9442a06cf55b8cc04360982dd725cb5 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 17 Jun 2009 13:45:08 +0900 Subject: [PATCH 0162/1172] Fix refcount leak and optimize list initialization. --- msgpack.c | 229 +++++++++++++++++++++++++----------------------------- setup.py | 2 +- unpack.h | 45 ++++++++--- 3 files changed, 142 insertions(+), 134 deletions(-) diff --git a/msgpack.c b/msgpack.c index 7e5c21f..ed0ac7e 100644 --- a/msgpack.c +++ b/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Tue Jun 16 01:57:05 2009 */ +/* Generated by Cython 0.11.1 on Wed Jun 17 13:44:30 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -103,9 +103,6 @@ #ifndef __cdecl #define __cdecl #endif - #ifndef __fastcall - #define __fastcall - #endif #else #define _USE_MATH_DEFINES #endif @@ -136,7 +133,6 @@ #include "string.h" #include "pack.h" #include "unpack.h" -#define __PYX_USE_C99_COMPLEX defined(_Complex_I) #ifdef __GNUC__ @@ -228,7 +224,6 @@ static const char * __pyx_cfilenm= __FILE__; static const char *__pyx_filename; static const char **__pyx_f; - #ifdef CYTHON_REFNANNY typedef struct { void (*INCREF)(void*, PyObject*, int); @@ -325,7 +320,7 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ /* Type declarations */ -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -343,7 +338,7 @@ struct __pyx_obj_7msgpack_Packer { PyObject *strm; }; -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":200 * return unpacks(packed) * * cdef class Unpacker: # <<<<<<<<<<<<<< @@ -356,7 +351,7 @@ struct __pyx_obj_7msgpack_Unpacker { }; -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -375,12 +370,10 @@ static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; static int __pyx_v_7msgpack_BUFF_SIZE; static PyObject *__pyx_k_1 = 0; static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ -#define __Pyx_MODULE_NAME "msgpack" -int __pyx_module_is_main_msgpack = 0; + +const char *__pyx_modulename = "msgpack"; /* Implementation of msgpack */ -static char __pyx_k___main__[] = "__main__"; -static PyObject *__pyx_kp___main__; static char __pyx_k___init__[] = "__init__"; static PyObject *__pyx_kp___init__; static char __pyx_k___del__[] = "__del__"; @@ -438,7 +431,7 @@ static PyObject *__pyx_kp_4; static char __pyx_k_3[] = "utf-8"; static char __pyx_k_4[] = "can't serialize %r"; -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":51 * cdef object strm * * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< @@ -500,7 +493,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return -1; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":52 * * def __init__(self, strm, int size=0): * if size <= 0: # <<<<<<<<<<<<<< @@ -510,7 +503,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __pyx_t_1 = (__pyx_v_size <= 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":53 * def __init__(self, strm, int size=0): * if size <= 0: * size = BUFF_SIZE # <<<<<<<<<<<<<< @@ -522,7 +515,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * } __pyx_L6:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":55 * size = BUFF_SIZE * * self.strm = strm # <<<<<<<<<<<<<< @@ -535,7 +528,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":56 * * self.strm = strm * self.buff = malloc(size) # <<<<<<<<<<<<<< @@ -544,7 +537,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":57 * self.strm = strm * self.buff = malloc(size) * self.allocated = size # <<<<<<<<<<<<<< @@ -553,7 +546,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":58 * self.buff = malloc(size) * self.allocated = size * self.length = 0 # <<<<<<<<<<<<<< @@ -562,7 +555,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":60 * self.length = 0 * * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< @@ -576,7 +569,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":62 * msgpack_packer_init(&self.pk, self, _packer_write) * * def __del__(self): # <<<<<<<<<<<<<< @@ -589,7 +582,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj PyObject *__pyx_r = NULL; __Pyx_SetupRefcountContext("__del__"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":63 * * def __del__(self): * free(self.buff); # <<<<<<<<<<<<<< @@ -604,7 +597,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":65 * free(self.buff); * * def flush(self): # <<<<<<<<<<<<<< @@ -622,7 +615,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("flush"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":67 * def flush(self): * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: # <<<<<<<<<<<<<< @@ -632,7 +625,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":68 * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< @@ -654,7 +647,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":69 * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 # <<<<<<<<<<<<<< @@ -666,7 +659,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec } __pyx_L5:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":70 * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< @@ -676,7 +669,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":71 * self.length = 0 * if hasattr(self.strm, 'flush'): * self.strm.flush() # <<<<<<<<<<<<<< @@ -707,7 +700,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":73 * self.strm.flush() * * def pack_list(self, len): # <<<<<<<<<<<<<< @@ -722,7 +715,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_list"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":86 * packer.pack(['foo', 'bar']) * """ * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< @@ -743,7 +736,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":88 * msgpack_pack_array(&self.pk, len) * * def pack_dict(self, len): # <<<<<<<<<<<<<< @@ -758,7 +751,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_dict"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":101 * packer.pack({'foo', 'bar'}) * """ * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< @@ -779,7 +772,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":103 * msgpack_pack_map(&self.pk, len) * * cdef __pack(self, object o): # <<<<<<<<<<<<<< @@ -813,7 +806,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":108 * cdef char* rawval * * if o is None: # <<<<<<<<<<<<<< @@ -823,7 +816,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = (__pyx_v_o == Py_None); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":109 * * if o is None: * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< @@ -834,7 +827,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":110 * if o is None: * msgpack_pack_nil(&self.pk) * elif o is True: # <<<<<<<<<<<<<< @@ -847,7 +840,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":111 * msgpack_pack_nil(&self.pk) * elif o is True: * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< @@ -858,7 +851,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":112 * elif o is True: * msgpack_pack_true(&self.pk) * elif o is False: # <<<<<<<<<<<<<< @@ -871,7 +864,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":113 * msgpack_pack_true(&self.pk) * elif o is False: * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< @@ -882,7 +875,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":114 * elif o is False: * msgpack_pack_false(&self.pk) * elif isinstance(o, long): # <<<<<<<<<<<<<< @@ -892,7 +885,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":115 * msgpack_pack_false(&self.pk) * elif isinstance(o, long): * intval = o # <<<<<<<<<<<<<< @@ -902,7 +895,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":116 * elif isinstance(o, long): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -913,7 +906,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":117 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): # <<<<<<<<<<<<<< @@ -923,7 +916,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":118 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): * intval = o # <<<<<<<<<<<<<< @@ -933,7 +926,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":119 * elif isinstance(o, int): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -944,7 +937,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":120 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< @@ -954,7 +947,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): * fval = o # <<<<<<<<<<<<<< @@ -964,7 +957,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_4 = __pyx_PyFloat_AsDouble(__pyx_v_o); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_fval = __pyx_t_4; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":122 * elif isinstance(o, float): * fval = o * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< @@ -975,7 +968,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":123 * fval = o * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< @@ -985,7 +978,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":124 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): * rawval = o # <<<<<<<<<<<<<< @@ -995,7 +988,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":125 * elif isinstance(o, str): * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1005,7 +998,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":126 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1017,7 +1010,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":127 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): # <<<<<<<<<<<<<< @@ -1027,7 +1020,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":128 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): * o = o.encode('utf-8') # <<<<<<<<<<<<<< @@ -1049,7 +1042,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_o = __pyx_t_8; __pyx_t_8 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":129 * elif isinstance(o, unicode): * o = o.encode('utf-8') * rawval = o # <<<<<<<<<<<<<< @@ -1059,7 +1052,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":130 * o = o.encode('utf-8') * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1069,7 +1062,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":131 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1081,7 +1074,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":132 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): # <<<<<<<<<<<<<< @@ -1091,7 +1084,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":133 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1101,7 +1094,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":134 * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): # <<<<<<<<<<<<<< @@ -1164,7 +1157,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_3 = 0; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":135 * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): * self.pack(k) # <<<<<<<<<<<<<< @@ -1184,7 +1177,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":136 * for k,v in o.iteritems(): * self.pack(k) * self.pack(v) # <<<<<<<<<<<<<< @@ -1208,7 +1201,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":137 * self.pack(k) * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< @@ -1224,7 +1217,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } if (__pyx_t_11) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":138 * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1234,7 +1227,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) * for v in o: # <<<<<<<<<<<<<< @@ -1266,7 +1259,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_v = __pyx_t_7; __pyx_t_7 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) * for v in o: * self.pack(v) # <<<<<<<<<<<<<< @@ -1291,7 +1284,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } /*else*/ { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":143 * else: * # TODO: Serialize with defalt() like simplejson. * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< @@ -1333,7 +1326,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -1394,7 +1387,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return NULL; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":146 * * def pack(self, obj, flush=True): * self.__pack(obj) # <<<<<<<<<<<<<< @@ -1405,7 +1398,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":147 * def pack(self, obj, flush=True): * self.__pack(obj) * if flush: # <<<<<<<<<<<<<< @@ -1415,7 +1408,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_2) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":148 * self.__pack(obj) * if flush: * self.flush() # <<<<<<<<<<<<<< @@ -1445,7 +1438,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":150 * self.flush() * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< @@ -1461,7 +1454,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("_packer_write"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":151 * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< @@ -1471,7 +1464,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":152 * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: * if packer.length > 0: # <<<<<<<<<<<<<< @@ -1481,7 +1474,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_packer->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":153 * if packer.length + l > packer.allocated: * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< @@ -1506,7 +1499,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L4:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":154 * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: # <<<<<<<<<<<<<< @@ -1516,7 +1509,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_l > 64); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":155 * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< @@ -1538,7 +1531,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":156 * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) * packer.length = 0 # <<<<<<<<<<<<<< @@ -1550,7 +1543,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":158 * packer.length = 0 * else: * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< @@ -1559,7 +1552,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":159 * else: * memcpy(packer.buff, b, l) * packer.length = l # <<<<<<<<<<<<<< @@ -1573,7 +1566,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":161 * packer.length = l * else: * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< @@ -1582,7 +1575,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":162 * else: * memcpy(packer.buff + packer.length, b, l) * packer.length += l # <<<<<<<<<<<<<< @@ -1593,7 +1586,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L3:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":163 * memcpy(packer.buff + packer.length, b, l) * packer.length += l * return 0 # <<<<<<<<<<<<<< @@ -1616,7 +1609,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":165 * return 0 * * def pack(object o, object stream): # <<<<<<<<<<<<<< @@ -1677,7 +1670,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_L4_argument_unpacking_done:; __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":166 * * def pack(object o, object stream): * packer = Packer(stream) # <<<<<<<<<<<<<< @@ -1696,7 +1689,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":167 * def pack(object o, object stream): * packer = Packer(stream) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1716,7 +1709,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":168 * packer = Packer(stream) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1745,7 +1738,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":170 * packer.flush() * * def packs(object o): # <<<<<<<<<<<<<< @@ -1767,7 +1760,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":171 * * def packs(object o): * buf = StringIO() # <<<<<<<<<<<<<< @@ -1783,7 +1776,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = __pyx_t_1; __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":172 * def packs(object o): * buf = StringIO() * packer = Packer(buf) # <<<<<<<<<<<<<< @@ -1802,7 +1795,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":173 * buf = StringIO() * packer = Packer(buf) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1822,7 +1815,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":174 * packer = Packer(buf) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1836,7 +1829,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":175 * packer.pack(o) * packer.flush() * return buf.getvalue() # <<<<<<<<<<<<<< @@ -1870,7 +1863,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":186 * * * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< @@ -1891,7 +1884,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":188 * def unpacks(object packed_bytes): * """Unpack packed_bytes to object. Returns unpacked object.""" * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< @@ -1901,7 +1894,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_p = __pyx_t_1; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":190 * cdef const_char_ptr p = packed_bytes * cdef template_context ctx * cdef size_t off = 0 # <<<<<<<<<<<<<< @@ -1910,7 +1903,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ __pyx_v_off = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":191 * cdef template_context ctx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< @@ -1919,7 +1912,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ template_init((&__pyx_v_ctx)); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":192 * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< @@ -1929,7 +1922,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":193 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) * return template_data(&ctx) # <<<<<<<<<<<<<< @@ -1955,7 +1948,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":195 * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< @@ -1975,7 +1968,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_self = __pyx_self; __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":197 * def unpack(object stream): * """unpack from stream.""" * packed = stream.read() # <<<<<<<<<<<<<< @@ -1991,7 +1984,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_v_packed = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":198 * """unpack from stream.""" * packed = stream.read() * return unpacks(packed) # <<<<<<<<<<<<<< @@ -2097,11 +2090,7 @@ static PyNumberMethods __pyx_tp_as_number_Packer = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else 0, /*nb_long*/ - #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2257,11 +2246,7 @@ static PyNumberMethods __pyx_tp_as_number_Unpacker = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else 0, /*nb_long*/ - #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2404,7 +2389,6 @@ static struct PyModuleDef __pyx_moduledef = { #endif static __Pyx_StringTabEntry __pyx_string_tab[] = { - {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, @@ -2496,9 +2480,6 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - if (__pyx_module_is_main_msgpack) { - if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - } /*--- Builtin init code ---*/ if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_skip_dispatch = 0; @@ -2522,7 +2503,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) /*--- Function import code ---*/ /*--- Execution code ---*/ - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":3 * # coding: utf-8 * * from cStringIO import StringIO # <<<<<<<<<<<<<< @@ -2543,7 +2524,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __Pyx_DECREF(__pyx_2); __pyx_2 = 0; __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":37 * * * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< @@ -2552,7 +2533,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) */ __pyx_v_7msgpack_BUFF_SIZE = 2048; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -2565,7 +2546,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_t_1 = 0; __Pyx_GIVEREF(__pyx_k_1); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":202 * cdef class Unpacker: * """Do nothing. This function is for symmetric to Packer""" * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< @@ -3368,14 +3349,14 @@ static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { return Py_INCREF(x), x; m = Py_TYPE(x)->tp_as_number; #if PY_VERSION_HEX < 0x03000000 - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Int(x); - } - else if (m && m->nb_long) { + if (m && m->nb_long) { name = "long"; res = PyNumber_Long(x); } + else if (m && m->nb_int) { + name = "int"; + res = PyNumber_Int(x); + } #else if (m && m->nb_int) { name = "int"; diff --git a/setup.py b/setup.py index e5651a0..4bb8693 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from distutils.core import setup, Extension version = '0.0.1' -msgpack_mod = Extension('msgpack', sources=['msgpack.c']) +msgpack_mod = Extension('msgpack', sources=['msgpack.c'], extra_compile_args=["-O3"]) desc = 'MessagePack serializer/desirializer.' long_desc = desc + """ diff --git a/unpack.h b/unpack.h index c98c19a..3e99123 100644 --- a/unpack.h +++ b/unpack.h @@ -20,7 +20,8 @@ #include "msgpack/unpack_define.h" typedef struct { - int reserved; + struct {unsigned int size, last} array_stack[MSGPACK_MAX_STACK_SIZE]; + int array_current; } unpack_user; @@ -42,7 +43,10 @@ struct template_context; typedef struct template_context template_context; static inline msgpack_unpack_object template_callback_root(unpack_user* u) -{ return NULL; } +{ + u->array_current = -1; + return NULL; +} static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) { *o = PyInt_FromLong((long)d); return 0; } @@ -52,8 +56,8 @@ static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_u static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { - if (d >= 0x80000000UL) { - *o = PyLong_FromUnsignedLongLong((unsigned long long)d); + if (d > LONG_MAX) { + *o = PyLong_FromUnsignedLong((unsigned long)d); } else { *o = PyInt_FromLong((long)d); } @@ -92,14 +96,32 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - /* TODO: use PyList_New(n). */ - *o = PyList_New(0); + if (n > 0) { + int cur = ++u->array_current; + u->array_stack[cur].size = n; + u->array_stack[cur].last = 0; + *o = PyList_New(n); + } + else { + *o = PyList_New(0); + } return 0; } static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - PyList_Append(*c, o); + int cur = u->array_current; + int n = u->array_stack[cur].size; + int last = u->array_stack[cur].last; + + PyList_SetItem(*c, last, o); + last++; + if (last >= n) { + u->array_current--; + } + else { + u->array_stack[cur].last = last; + } return 0; } @@ -112,13 +134,18 @@ static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { PyDict_SetItem(*c, k, v); - return 0; + Py_DECREF(k); + Py_DECREF(v); + return 0; } static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { *o = PyString_FromStringAndSize(p, l); - return 0; + if (l < 16) { // without foundation + PyString_InternInPlace(o); + } + return 0; } #include "msgpack/unpack_template.h" From 0d14239c2197b616196a535c9aeb818da1040e9d Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 09:51:24 +0900 Subject: [PATCH 0163/1172] Optimize to parsing data that has a number of same short raw field. --- python/{msgpack.c => msgpack.cpp} | 229 ++++++++++++++++-------------- python/setup.py | 2 +- python/unpack.h | 38 ++++- 3 files changed, 158 insertions(+), 111 deletions(-) rename python/{msgpack.c => msgpack.cpp} (94%) diff --git a/python/msgpack.c b/python/msgpack.cpp similarity index 94% rename from python/msgpack.c rename to python/msgpack.cpp index ed0ac7e..760f0c0 100644 --- a/python/msgpack.c +++ b/python/msgpack.cpp @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.1 on Wed Jun 17 13:44:30 2009 */ +/* Generated by Cython 0.11.2 on Mon Jun 22 02:56:08 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -103,6 +103,9 @@ #ifndef __cdecl #define __cdecl #endif + #ifndef __fastcall + #define __fastcall + #endif #else #define _USE_MATH_DEFINES #endif @@ -133,6 +136,7 @@ #include "string.h" #include "pack.h" #include "unpack.h" +#define __PYX_USE_C99_COMPLEX defined(_Complex_I) #ifdef __GNUC__ @@ -224,6 +228,7 @@ static const char * __pyx_cfilenm= __FILE__; static const char *__pyx_filename; static const char **__pyx_f; + #ifdef CYTHON_REFNANNY typedef struct { void (*INCREF)(void*, PyObject*, int); @@ -320,7 +325,7 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ /* Type declarations */ -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -338,7 +343,7 @@ struct __pyx_obj_7msgpack_Packer { PyObject *strm; }; -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":200 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 * return unpacks(packed) * * cdef class Unpacker: # <<<<<<<<<<<<<< @@ -351,7 +356,7 @@ struct __pyx_obj_7msgpack_Unpacker { }; -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -370,10 +375,12 @@ static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; static int __pyx_v_7msgpack_BUFF_SIZE; static PyObject *__pyx_k_1 = 0; static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ - -const char *__pyx_modulename = "msgpack"; +#define __Pyx_MODULE_NAME "msgpack" +int __pyx_module_is_main_msgpack = 0; /* Implementation of msgpack */ +static char __pyx_k___main__[] = "__main__"; +static PyObject *__pyx_kp___main__; static char __pyx_k___init__[] = "__init__"; static PyObject *__pyx_kp___init__; static char __pyx_k___del__[] = "__del__"; @@ -431,7 +438,7 @@ static PyObject *__pyx_kp_4; static char __pyx_k_3[] = "utf-8"; static char __pyx_k_4[] = "can't serialize %r"; -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":51 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 * cdef object strm * * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< @@ -493,7 +500,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return -1; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":52 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 * * def __init__(self, strm, int size=0): * if size <= 0: # <<<<<<<<<<<<<< @@ -503,7 +510,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __pyx_t_1 = (__pyx_v_size <= 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":53 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 * def __init__(self, strm, int size=0): * if size <= 0: * size = BUFF_SIZE # <<<<<<<<<<<<<< @@ -515,7 +522,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * } __pyx_L6:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":55 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 * size = BUFF_SIZE * * self.strm = strm # <<<<<<<<<<<<<< @@ -528,7 +535,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":56 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 * * self.strm = strm * self.buff = malloc(size) # <<<<<<<<<<<<<< @@ -537,7 +544,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":57 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 * self.strm = strm * self.buff = malloc(size) * self.allocated = size # <<<<<<<<<<<<<< @@ -546,7 +553,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":58 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 * self.buff = malloc(size) * self.allocated = size * self.length = 0 # <<<<<<<<<<<<<< @@ -555,7 +562,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":60 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 * self.length = 0 * * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< @@ -569,7 +576,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":62 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 * msgpack_packer_init(&self.pk, self, _packer_write) * * def __del__(self): # <<<<<<<<<<<<<< @@ -582,7 +589,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj PyObject *__pyx_r = NULL; __Pyx_SetupRefcountContext("__del__"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":63 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 * * def __del__(self): * free(self.buff); # <<<<<<<<<<<<<< @@ -597,7 +604,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":65 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 * free(self.buff); * * def flush(self): # <<<<<<<<<<<<<< @@ -615,7 +622,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("flush"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":67 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 * def flush(self): * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: # <<<<<<<<<<<<<< @@ -625,7 +632,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":68 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< @@ -647,7 +654,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":69 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 # <<<<<<<<<<<<<< @@ -659,7 +666,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec } __pyx_L5:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":70 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< @@ -669,7 +676,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":71 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 * self.length = 0 * if hasattr(self.strm, 'flush'): * self.strm.flush() # <<<<<<<<<<<<<< @@ -700,7 +707,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":73 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 * self.strm.flush() * * def pack_list(self, len): # <<<<<<<<<<<<<< @@ -715,7 +722,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_list"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":86 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 * packer.pack(['foo', 'bar']) * """ * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< @@ -736,7 +743,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":88 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 * msgpack_pack_array(&self.pk, len) * * def pack_dict(self, len): # <<<<<<<<<<<<<< @@ -751,7 +758,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_dict"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":101 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 * packer.pack({'foo', 'bar'}) * """ * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< @@ -772,7 +779,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":103 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 * msgpack_pack_map(&self.pk, len) * * cdef __pack(self, object o): # <<<<<<<<<<<<<< @@ -806,7 +813,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":108 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 * cdef char* rawval * * if o is None: # <<<<<<<<<<<<<< @@ -816,7 +823,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = (__pyx_v_o == Py_None); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":109 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 * * if o is None: * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< @@ -827,7 +834,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":110 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 * if o is None: * msgpack_pack_nil(&self.pk) * elif o is True: # <<<<<<<<<<<<<< @@ -840,7 +847,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":111 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 * msgpack_pack_nil(&self.pk) * elif o is True: * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< @@ -851,7 +858,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":112 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 * elif o is True: * msgpack_pack_true(&self.pk) * elif o is False: # <<<<<<<<<<<<<< @@ -864,7 +871,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":113 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 * msgpack_pack_true(&self.pk) * elif o is False: * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< @@ -875,7 +882,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":114 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 * elif o is False: * msgpack_pack_false(&self.pk) * elif isinstance(o, long): # <<<<<<<<<<<<<< @@ -885,7 +892,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":115 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 * msgpack_pack_false(&self.pk) * elif isinstance(o, long): * intval = o # <<<<<<<<<<<<<< @@ -895,7 +902,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":116 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 * elif isinstance(o, long): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -906,7 +913,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":117 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): # <<<<<<<<<<<<<< @@ -916,7 +923,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":118 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): * intval = o # <<<<<<<<<<<<<< @@ -926,7 +933,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":119 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 * elif isinstance(o, int): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -937,7 +944,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":120 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< @@ -947,7 +954,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":121 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): * fval = o # <<<<<<<<<<<<<< @@ -957,7 +964,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_4 = __pyx_PyFloat_AsDouble(__pyx_v_o); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_fval = __pyx_t_4; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":122 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 * elif isinstance(o, float): * fval = o * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< @@ -968,7 +975,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":123 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 * fval = o * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< @@ -978,7 +985,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":124 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): * rawval = o # <<<<<<<<<<<<<< @@ -988,7 +995,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":125 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 * elif isinstance(o, str): * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -998,7 +1005,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":126 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1010,7 +1017,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":127 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): # <<<<<<<<<<<<<< @@ -1020,7 +1027,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":128 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): * o = o.encode('utf-8') # <<<<<<<<<<<<<< @@ -1042,7 +1049,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_o = __pyx_t_8; __pyx_t_8 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":129 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 * elif isinstance(o, unicode): * o = o.encode('utf-8') * rawval = o # <<<<<<<<<<<<<< @@ -1052,7 +1059,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":130 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 * o = o.encode('utf-8') * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1062,7 +1069,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":131 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1074,7 +1081,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":132 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): # <<<<<<<<<<<<<< @@ -1084,7 +1091,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":133 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1094,7 +1101,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":134 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): # <<<<<<<<<<<<<< @@ -1157,7 +1164,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_3 = 0; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":135 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): * self.pack(k) # <<<<<<<<<<<<<< @@ -1177,7 +1184,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":136 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 * for k,v in o.iteritems(): * self.pack(k) * self.pack(v) # <<<<<<<<<<<<<< @@ -1201,7 +1208,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":137 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 * self.pack(k) * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< @@ -1217,7 +1224,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } if (__pyx_t_11) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":138 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1227,7 +1234,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":139 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) * for v in o: # <<<<<<<<<<<<<< @@ -1259,7 +1266,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_v = __pyx_t_7; __pyx_t_7 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":140 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) * for v in o: * self.pack(v) # <<<<<<<<<<<<<< @@ -1284,7 +1291,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } /*else*/ { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":143 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 * else: * # TODO: Serialize with defalt() like simplejson. * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< @@ -1326,7 +1333,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -1387,7 +1394,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return NULL; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":146 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 * * def pack(self, obj, flush=True): * self.__pack(obj) # <<<<<<<<<<<<<< @@ -1398,7 +1405,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":147 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 * def pack(self, obj, flush=True): * self.__pack(obj) * if flush: # <<<<<<<<<<<<<< @@ -1408,7 +1415,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_2) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":148 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 * self.__pack(obj) * if flush: * self.flush() # <<<<<<<<<<<<<< @@ -1438,7 +1445,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":150 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 * self.flush() * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< @@ -1454,7 +1461,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("_packer_write"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":151 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< @@ -1464,7 +1471,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":152 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: * if packer.length > 0: # <<<<<<<<<<<<<< @@ -1474,7 +1481,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_packer->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":153 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 * if packer.length + l > packer.allocated: * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< @@ -1499,7 +1506,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L4:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":154 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: # <<<<<<<<<<<<<< @@ -1509,7 +1516,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_l > 64); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":155 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< @@ -1531,7 +1538,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":156 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) * packer.length = 0 # <<<<<<<<<<<<<< @@ -1543,7 +1550,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":158 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 * packer.length = 0 * else: * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< @@ -1552,7 +1559,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":159 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 * else: * memcpy(packer.buff, b, l) * packer.length = l # <<<<<<<<<<<<<< @@ -1566,7 +1573,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":161 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 * packer.length = l * else: * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< @@ -1575,7 +1582,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":162 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 * else: * memcpy(packer.buff + packer.length, b, l) * packer.length += l # <<<<<<<<<<<<<< @@ -1586,7 +1593,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L3:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":163 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 * memcpy(packer.buff + packer.length, b, l) * packer.length += l * return 0 # <<<<<<<<<<<<<< @@ -1609,7 +1616,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":165 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 * return 0 * * def pack(object o, object stream): # <<<<<<<<<<<<<< @@ -1670,7 +1677,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_L4_argument_unpacking_done:; __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":166 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 * * def pack(object o, object stream): * packer = Packer(stream) # <<<<<<<<<<<<<< @@ -1689,7 +1696,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":167 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 * def pack(object o, object stream): * packer = Packer(stream) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1709,7 +1716,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":168 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 * packer = Packer(stream) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1738,7 +1745,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":170 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 * packer.flush() * * def packs(object o): # <<<<<<<<<<<<<< @@ -1760,7 +1767,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":171 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 * * def packs(object o): * buf = StringIO() # <<<<<<<<<<<<<< @@ -1776,7 +1783,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = __pyx_t_1; __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":172 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 * def packs(object o): * buf = StringIO() * packer = Packer(buf) # <<<<<<<<<<<<<< @@ -1795,7 +1802,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":173 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 * buf = StringIO() * packer = Packer(buf) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1815,7 +1822,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":174 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 * packer = Packer(buf) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1829,7 +1836,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":175 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 * packer.pack(o) * packer.flush() * return buf.getvalue() # <<<<<<<<<<<<<< @@ -1863,7 +1870,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":186 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 * * * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< @@ -1884,7 +1891,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":188 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 * def unpacks(object packed_bytes): * """Unpack packed_bytes to object. Returns unpacked object.""" * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< @@ -1894,7 +1901,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_p = __pyx_t_1; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":190 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 * cdef const_char_ptr p = packed_bytes * cdef template_context ctx * cdef size_t off = 0 # <<<<<<<<<<<<<< @@ -1903,7 +1910,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ __pyx_v_off = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":191 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 * cdef template_context ctx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< @@ -1912,7 +1919,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ template_init((&__pyx_v_ctx)); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":192 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< @@ -1922,7 +1929,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":193 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) * return template_data(&ctx) # <<<<<<<<<<<<<< @@ -1948,7 +1955,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":195 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< @@ -1968,7 +1975,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_self = __pyx_self; __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":197 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 * def unpack(object stream): * """unpack from stream.""" * packed = stream.read() # <<<<<<<<<<<<<< @@ -1984,7 +1991,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_v_packed = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":198 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 * """unpack from stream.""" * packed = stream.read() * return unpacks(packed) # <<<<<<<<<<<<<< @@ -2090,7 +2097,11 @@ static PyNumberMethods __pyx_tp_as_number_Packer = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else 0, /*nb_long*/ + #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2246,7 +2257,11 @@ static PyNumberMethods __pyx_tp_as_number_Unpacker = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else 0, /*nb_long*/ + #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2389,6 +2404,7 @@ static struct PyModuleDef __pyx_moduledef = { #endif static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, @@ -2480,6 +2496,9 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + if (__pyx_module_is_main_msgpack) { + if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + } /*--- Builtin init code ---*/ if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_skip_dispatch = 0; @@ -2503,7 +2522,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) /*--- Function import code ---*/ /*--- Execution code ---*/ - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":3 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 * # coding: utf-8 * * from cStringIO import StringIO # <<<<<<<<<<<<<< @@ -2524,7 +2543,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __Pyx_DECREF(__pyx_2); __pyx_2 = 0; __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":37 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 * * * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< @@ -2533,7 +2552,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) */ __pyx_v_7msgpack_BUFF_SIZE = 2048; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -2546,7 +2565,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_t_1 = 0; __Pyx_GIVEREF(__pyx_k_1); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":202 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 * cdef class Unpacker: * """Do nothing. This function is for symmetric to Packer""" * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< @@ -3349,14 +3368,14 @@ static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { return Py_INCREF(x), x; m = Py_TYPE(x)->tp_as_number; #if PY_VERSION_HEX < 0x03000000 - if (m && m->nb_long) { - name = "long"; - res = PyNumber_Long(x); - } - else if (m && m->nb_int) { + if (m && m->nb_int) { name = "int"; res = PyNumber_Int(x); } + else if (m && m->nb_long) { + name = "long"; + res = PyNumber_Long(x); + } #else if (m && m->nb_int) { name = "int"; diff --git a/python/setup.py b/python/setup.py index 4bb8693..f4c84dc 100644 --- a/python/setup.py +++ b/python/setup.py @@ -2,7 +2,7 @@ from distutils.core import setup, Extension version = '0.0.1' -msgpack_mod = Extension('msgpack', sources=['msgpack.c'], extra_compile_args=["-O3"]) +msgpack_mod = Extension('msgpack', sources=['msgpack.cpp']) desc = 'MessagePack serializer/desirializer.' long_desc = desc + """ diff --git a/python/unpack.h b/python/unpack.h index 3e99123..694e816 100644 --- a/python/unpack.h +++ b/python/unpack.h @@ -16,12 +16,28 @@ * limitations under the License. */ +#include +#include + #define MSGPACK_MAX_STACK_SIZE (1024) #include "msgpack/unpack_define.h" -typedef struct { - struct {unsigned int size, last} array_stack[MSGPACK_MAX_STACK_SIZE]; +using namespace std; + +typedef struct unpack_user { + struct array_stack_type{unsigned int size, last;}; + array_stack_type array_stack[MSGPACK_MAX_STACK_SIZE]; int array_current; + + map str_cache; + + ~unpack_user() { + map::iterator it, itend; + itend = str_cache.end(); + for (it = str_cache.begin(); it != itend; ++it) { + Py_DECREF(it->second); + } + } } unpack_user; @@ -141,9 +157,21 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { - *o = PyString_FromStringAndSize(p, l); - if (l < 16) { // without foundation - PyString_InternInPlace(o); + if (l < 16) { + string s(p, l); + map::iterator it = u->str_cache.find(s); + if (it != u->str_cache.end()) { + *o = it->second; + Py_INCREF(*o); + } + else { + *o = PyString_FromStringAndSize(p, l); + Py_INCREF(*o); + u->str_cache[s] = *o; + } + } + else { + *o = PyString_FromStringAndSize(p, l); } return 0; } From dee7b2092474d00090f22db95623f2ac134eada2 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 09:51:24 +0900 Subject: [PATCH 0164/1172] Optimize to parsing data that has a number of same short raw field. --- msgpack.c => msgpack.cpp | 229 +++++++++++++++++++++------------------ setup.py | 2 +- unpack.h | 38 ++++++- 3 files changed, 158 insertions(+), 111 deletions(-) rename msgpack.c => msgpack.cpp (94%) diff --git a/msgpack.c b/msgpack.cpp similarity index 94% rename from msgpack.c rename to msgpack.cpp index ed0ac7e..760f0c0 100644 --- a/msgpack.c +++ b/msgpack.cpp @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.1 on Wed Jun 17 13:44:30 2009 */ +/* Generated by Cython 0.11.2 on Mon Jun 22 02:56:08 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -103,6 +103,9 @@ #ifndef __cdecl #define __cdecl #endif + #ifndef __fastcall + #define __fastcall + #endif #else #define _USE_MATH_DEFINES #endif @@ -133,6 +136,7 @@ #include "string.h" #include "pack.h" #include "unpack.h" +#define __PYX_USE_C99_COMPLEX defined(_Complex_I) #ifdef __GNUC__ @@ -224,6 +228,7 @@ static const char * __pyx_cfilenm= __FILE__; static const char *__pyx_filename; static const char **__pyx_f; + #ifdef CYTHON_REFNANNY typedef struct { void (*INCREF)(void*, PyObject*, int); @@ -320,7 +325,7 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ /* Type declarations */ -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -338,7 +343,7 @@ struct __pyx_obj_7msgpack_Packer { PyObject *strm; }; -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":200 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 * return unpacks(packed) * * cdef class Unpacker: # <<<<<<<<<<<<<< @@ -351,7 +356,7 @@ struct __pyx_obj_7msgpack_Unpacker { }; -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -370,10 +375,12 @@ static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; static int __pyx_v_7msgpack_BUFF_SIZE; static PyObject *__pyx_k_1 = 0; static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ - -const char *__pyx_modulename = "msgpack"; +#define __Pyx_MODULE_NAME "msgpack" +int __pyx_module_is_main_msgpack = 0; /* Implementation of msgpack */ +static char __pyx_k___main__[] = "__main__"; +static PyObject *__pyx_kp___main__; static char __pyx_k___init__[] = "__init__"; static PyObject *__pyx_kp___init__; static char __pyx_k___del__[] = "__del__"; @@ -431,7 +438,7 @@ static PyObject *__pyx_kp_4; static char __pyx_k_3[] = "utf-8"; static char __pyx_k_4[] = "can't serialize %r"; -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":51 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 * cdef object strm * * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< @@ -493,7 +500,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return -1; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":52 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 * * def __init__(self, strm, int size=0): * if size <= 0: # <<<<<<<<<<<<<< @@ -503,7 +510,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __pyx_t_1 = (__pyx_v_size <= 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":53 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 * def __init__(self, strm, int size=0): * if size <= 0: * size = BUFF_SIZE # <<<<<<<<<<<<<< @@ -515,7 +522,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * } __pyx_L6:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":55 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 * size = BUFF_SIZE * * self.strm = strm # <<<<<<<<<<<<<< @@ -528,7 +535,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":56 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 * * self.strm = strm * self.buff = malloc(size) # <<<<<<<<<<<<<< @@ -537,7 +544,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":57 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 * self.strm = strm * self.buff = malloc(size) * self.allocated = size # <<<<<<<<<<<<<< @@ -546,7 +553,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":58 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 * self.buff = malloc(size) * self.allocated = size * self.length = 0 # <<<<<<<<<<<<<< @@ -555,7 +562,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":60 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 * self.length = 0 * * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< @@ -569,7 +576,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":62 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 * msgpack_packer_init(&self.pk, self, _packer_write) * * def __del__(self): # <<<<<<<<<<<<<< @@ -582,7 +589,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj PyObject *__pyx_r = NULL; __Pyx_SetupRefcountContext("__del__"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":63 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 * * def __del__(self): * free(self.buff); # <<<<<<<<<<<<<< @@ -597,7 +604,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":65 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 * free(self.buff); * * def flush(self): # <<<<<<<<<<<<<< @@ -615,7 +622,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("flush"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":67 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 * def flush(self): * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: # <<<<<<<<<<<<<< @@ -625,7 +632,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":68 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< @@ -647,7 +654,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":69 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 # <<<<<<<<<<<<<< @@ -659,7 +666,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec } __pyx_L5:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":70 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< @@ -669,7 +676,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":71 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 * self.length = 0 * if hasattr(self.strm, 'flush'): * self.strm.flush() # <<<<<<<<<<<<<< @@ -700,7 +707,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":73 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 * self.strm.flush() * * def pack_list(self, len): # <<<<<<<<<<<<<< @@ -715,7 +722,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_list"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":86 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 * packer.pack(['foo', 'bar']) * """ * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< @@ -736,7 +743,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":88 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 * msgpack_pack_array(&self.pk, len) * * def pack_dict(self, len): # <<<<<<<<<<<<<< @@ -751,7 +758,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_dict"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":101 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 * packer.pack({'foo', 'bar'}) * """ * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< @@ -772,7 +779,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":103 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 * msgpack_pack_map(&self.pk, len) * * cdef __pack(self, object o): # <<<<<<<<<<<<<< @@ -806,7 +813,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":108 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 * cdef char* rawval * * if o is None: # <<<<<<<<<<<<<< @@ -816,7 +823,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = (__pyx_v_o == Py_None); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":109 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 * * if o is None: * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< @@ -827,7 +834,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":110 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 * if o is None: * msgpack_pack_nil(&self.pk) * elif o is True: # <<<<<<<<<<<<<< @@ -840,7 +847,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":111 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 * msgpack_pack_nil(&self.pk) * elif o is True: * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< @@ -851,7 +858,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":112 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 * elif o is True: * msgpack_pack_true(&self.pk) * elif o is False: # <<<<<<<<<<<<<< @@ -864,7 +871,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":113 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 * msgpack_pack_true(&self.pk) * elif o is False: * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< @@ -875,7 +882,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":114 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 * elif o is False: * msgpack_pack_false(&self.pk) * elif isinstance(o, long): # <<<<<<<<<<<<<< @@ -885,7 +892,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":115 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 * msgpack_pack_false(&self.pk) * elif isinstance(o, long): * intval = o # <<<<<<<<<<<<<< @@ -895,7 +902,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":116 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 * elif isinstance(o, long): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -906,7 +913,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":117 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): # <<<<<<<<<<<<<< @@ -916,7 +923,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":118 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): * intval = o # <<<<<<<<<<<<<< @@ -926,7 +933,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":119 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 * elif isinstance(o, int): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -937,7 +944,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":120 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< @@ -947,7 +954,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":121 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): * fval = o # <<<<<<<<<<<<<< @@ -957,7 +964,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_4 = __pyx_PyFloat_AsDouble(__pyx_v_o); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_fval = __pyx_t_4; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":122 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 * elif isinstance(o, float): * fval = o * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< @@ -968,7 +975,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":123 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 * fval = o * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< @@ -978,7 +985,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":124 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): * rawval = o # <<<<<<<<<<<<<< @@ -988,7 +995,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":125 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 * elif isinstance(o, str): * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -998,7 +1005,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":126 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1010,7 +1017,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":127 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): # <<<<<<<<<<<<<< @@ -1020,7 +1027,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":128 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): * o = o.encode('utf-8') # <<<<<<<<<<<<<< @@ -1042,7 +1049,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_o = __pyx_t_8; __pyx_t_8 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":129 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 * elif isinstance(o, unicode): * o = o.encode('utf-8') * rawval = o # <<<<<<<<<<<<<< @@ -1052,7 +1059,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":130 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 * o = o.encode('utf-8') * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1062,7 +1069,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":131 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1074,7 +1081,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":132 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): # <<<<<<<<<<<<<< @@ -1084,7 +1091,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":133 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1094,7 +1101,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":134 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): # <<<<<<<<<<<<<< @@ -1157,7 +1164,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_3 = 0; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":135 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): * self.pack(k) # <<<<<<<<<<<<<< @@ -1177,7 +1184,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":136 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 * for k,v in o.iteritems(): * self.pack(k) * self.pack(v) # <<<<<<<<<<<<<< @@ -1201,7 +1208,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":137 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 * self.pack(k) * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< @@ -1217,7 +1224,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } if (__pyx_t_11) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":138 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1227,7 +1234,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":139 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) * for v in o: # <<<<<<<<<<<<<< @@ -1259,7 +1266,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_v = __pyx_t_7; __pyx_t_7 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":140 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) * for v in o: * self.pack(v) # <<<<<<<<<<<<<< @@ -1284,7 +1291,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } /*else*/ { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":143 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 * else: * # TODO: Serialize with defalt() like simplejson. * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< @@ -1326,7 +1333,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -1387,7 +1394,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return NULL; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":146 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 * * def pack(self, obj, flush=True): * self.__pack(obj) # <<<<<<<<<<<<<< @@ -1398,7 +1405,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":147 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 * def pack(self, obj, flush=True): * self.__pack(obj) * if flush: # <<<<<<<<<<<<<< @@ -1408,7 +1415,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_2) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":148 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 * self.__pack(obj) * if flush: * self.flush() # <<<<<<<<<<<<<< @@ -1438,7 +1445,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":150 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 * self.flush() * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< @@ -1454,7 +1461,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("_packer_write"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":151 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< @@ -1464,7 +1471,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":152 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: * if packer.length > 0: # <<<<<<<<<<<<<< @@ -1474,7 +1481,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_packer->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":153 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 * if packer.length + l > packer.allocated: * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< @@ -1499,7 +1506,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L4:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":154 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: # <<<<<<<<<<<<<< @@ -1509,7 +1516,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_l > 64); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":155 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< @@ -1531,7 +1538,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":156 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) * packer.length = 0 # <<<<<<<<<<<<<< @@ -1543,7 +1550,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":158 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 * packer.length = 0 * else: * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< @@ -1552,7 +1559,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":159 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 * else: * memcpy(packer.buff, b, l) * packer.length = l # <<<<<<<<<<<<<< @@ -1566,7 +1573,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":161 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 * packer.length = l * else: * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< @@ -1575,7 +1582,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":162 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 * else: * memcpy(packer.buff + packer.length, b, l) * packer.length += l # <<<<<<<<<<<<<< @@ -1586,7 +1593,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L3:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":163 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 * memcpy(packer.buff + packer.length, b, l) * packer.length += l * return 0 # <<<<<<<<<<<<<< @@ -1609,7 +1616,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":165 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 * return 0 * * def pack(object o, object stream): # <<<<<<<<<<<<<< @@ -1670,7 +1677,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_L4_argument_unpacking_done:; __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":166 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 * * def pack(object o, object stream): * packer = Packer(stream) # <<<<<<<<<<<<<< @@ -1689,7 +1696,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":167 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 * def pack(object o, object stream): * packer = Packer(stream) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1709,7 +1716,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":168 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 * packer = Packer(stream) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1738,7 +1745,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":170 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 * packer.flush() * * def packs(object o): # <<<<<<<<<<<<<< @@ -1760,7 +1767,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":171 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 * * def packs(object o): * buf = StringIO() # <<<<<<<<<<<<<< @@ -1776,7 +1783,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = __pyx_t_1; __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":172 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 * def packs(object o): * buf = StringIO() * packer = Packer(buf) # <<<<<<<<<<<<<< @@ -1795,7 +1802,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":173 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 * buf = StringIO() * packer = Packer(buf) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1815,7 +1822,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":174 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 * packer = Packer(buf) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1829,7 +1836,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":175 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 * packer.pack(o) * packer.flush() * return buf.getvalue() # <<<<<<<<<<<<<< @@ -1863,7 +1870,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":186 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 * * * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< @@ -1884,7 +1891,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":188 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 * def unpacks(object packed_bytes): * """Unpack packed_bytes to object. Returns unpacked object.""" * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< @@ -1894,7 +1901,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_p = __pyx_t_1; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":190 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 * cdef const_char_ptr p = packed_bytes * cdef template_context ctx * cdef size_t off = 0 # <<<<<<<<<<<<<< @@ -1903,7 +1910,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ __pyx_v_off = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":191 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 * cdef template_context ctx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< @@ -1912,7 +1919,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ template_init((&__pyx_v_ctx)); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":192 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< @@ -1922,7 +1929,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":193 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) * return template_data(&ctx) # <<<<<<<<<<<<<< @@ -1948,7 +1955,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":195 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< @@ -1968,7 +1975,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_self = __pyx_self; __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":197 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 * def unpack(object stream): * """unpack from stream.""" * packed = stream.read() # <<<<<<<<<<<<<< @@ -1984,7 +1991,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_v_packed = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":198 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 * """unpack from stream.""" * packed = stream.read() * return unpacks(packed) # <<<<<<<<<<<<<< @@ -2090,7 +2097,11 @@ static PyNumberMethods __pyx_tp_as_number_Packer = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else 0, /*nb_long*/ + #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2246,7 +2257,11 @@ static PyNumberMethods __pyx_tp_as_number_Unpacker = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else 0, /*nb_long*/ + #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2389,6 +2404,7 @@ static struct PyModuleDef __pyx_moduledef = { #endif static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, @@ -2480,6 +2496,9 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + if (__pyx_module_is_main_msgpack) { + if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + } /*--- Builtin init code ---*/ if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_skip_dispatch = 0; @@ -2503,7 +2522,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) /*--- Function import code ---*/ /*--- Execution code ---*/ - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":3 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 * # coding: utf-8 * * from cStringIO import StringIO # <<<<<<<<<<<<<< @@ -2524,7 +2543,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __Pyx_DECREF(__pyx_2); __pyx_2 = 0; __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":37 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 * * * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< @@ -2533,7 +2552,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) */ __pyx_v_7msgpack_BUFF_SIZE = 2048; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -2546,7 +2565,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_t_1 = 0; __Pyx_GIVEREF(__pyx_k_1); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":202 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 * cdef class Unpacker: * """Do nothing. This function is for symmetric to Packer""" * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< @@ -3349,14 +3368,14 @@ static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { return Py_INCREF(x), x; m = Py_TYPE(x)->tp_as_number; #if PY_VERSION_HEX < 0x03000000 - if (m && m->nb_long) { - name = "long"; - res = PyNumber_Long(x); - } - else if (m && m->nb_int) { + if (m && m->nb_int) { name = "int"; res = PyNumber_Int(x); } + else if (m && m->nb_long) { + name = "long"; + res = PyNumber_Long(x); + } #else if (m && m->nb_int) { name = "int"; diff --git a/setup.py b/setup.py index 4bb8693..f4c84dc 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from distutils.core import setup, Extension version = '0.0.1' -msgpack_mod = Extension('msgpack', sources=['msgpack.c'], extra_compile_args=["-O3"]) +msgpack_mod = Extension('msgpack', sources=['msgpack.cpp']) desc = 'MessagePack serializer/desirializer.' long_desc = desc + """ diff --git a/unpack.h b/unpack.h index 3e99123..694e816 100644 --- a/unpack.h +++ b/unpack.h @@ -16,12 +16,28 @@ * limitations under the License. */ +#include +#include + #define MSGPACK_MAX_STACK_SIZE (1024) #include "msgpack/unpack_define.h" -typedef struct { - struct {unsigned int size, last} array_stack[MSGPACK_MAX_STACK_SIZE]; +using namespace std; + +typedef struct unpack_user { + struct array_stack_type{unsigned int size, last;}; + array_stack_type array_stack[MSGPACK_MAX_STACK_SIZE]; int array_current; + + map str_cache; + + ~unpack_user() { + map::iterator it, itend; + itend = str_cache.end(); + for (it = str_cache.begin(); it != itend; ++it) { + Py_DECREF(it->second); + } + } } unpack_user; @@ -141,9 +157,21 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { - *o = PyString_FromStringAndSize(p, l); - if (l < 16) { // without foundation - PyString_InternInPlace(o); + if (l < 16) { + string s(p, l); + map::iterator it = u->str_cache.find(s); + if (it != u->str_cache.end()) { + *o = it->second; + Py_INCREF(*o); + } + else { + *o = PyString_FromStringAndSize(p, l); + Py_INCREF(*o); + u->str_cache[s] = *o; + } + } + else { + *o = PyString_FromStringAndSize(p, l); } return 0; } From 075081a521c8287759847f1aa846b88fef1f0046 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 14:28:04 +0900 Subject: [PATCH 0165/1172] Fix manifest. --- python/MANIFEST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/MANIFEST b/python/MANIFEST index dc042ae..dd0c7d2 100644 --- a/python/MANIFEST +++ b/python/MANIFEST @@ -1,4 +1,4 @@ -msgpack.c +msgpack.cpp setup.py pack.h unpack.h From 07806c92630480337fe9dcf3d79a46368f40bf5c Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 14:28:04 +0900 Subject: [PATCH 0166/1172] Fix manifest. --- MANIFEST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST b/MANIFEST index dc042ae..dd0c7d2 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1,4 +1,4 @@ -msgpack.c +msgpack.cpp setup.py pack.h unpack.h From 87f5df1503e70efd2bb9e9565259f1ed37a45933 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 15:59:02 +0900 Subject: [PATCH 0167/1172] Use std::stack. --- python/msgpack/unpack.h | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 694e816..daeb54c 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -18,17 +18,21 @@ #include #include +#include #define MSGPACK_MAX_STACK_SIZE (1024) #include "msgpack/unpack_define.h" using namespace std; -typedef struct unpack_user { - struct array_stack_type{unsigned int size, last;}; - array_stack_type array_stack[MSGPACK_MAX_STACK_SIZE]; - int array_current; - +struct array_context { + unsigned int size; + unsigned int last; + stack_item(unsigned int size) : size(size), last(0) + {} +}; +struct unpack_user { + stack array_stack; map str_cache; ~unpack_user() { @@ -38,7 +42,7 @@ typedef struct unpack_user { Py_DECREF(it->second); } } -} unpack_user; +}; #define msgpack_unpack_struct(name) \ @@ -60,7 +64,6 @@ typedef struct template_context template_context; static inline msgpack_unpack_object template_callback_root(unpack_user* u) { - u->array_current = -1; return NULL; } @@ -113,9 +116,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - int cur = ++u->array_current; - u->array_stack[cur].size = n; - u->array_stack[cur].last = 0; + u->array_stack.push(stack_item(n)); *o = PyList_New(n); } else { @@ -126,17 +127,13 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - int cur = u->array_current; - int n = u->array_stack[cur].size; - int last = u->array_stack[cur].last; + unsigned int n = u->array_stack.top().size; + unsigned int &last = u->array_stack.top().last; PyList_SetItem(*c, last, o); last++; if (last >= n) { - u->array_current--; - } - else { - u->array_stack[cur].last = last; + u->array_stack.pop(); } return 0; } From d44108c70977c859a96caa8847ec2514f2313fdd Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 15:59:02 +0900 Subject: [PATCH 0168/1172] Use std::stack. --- msgpack/unpack.h | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 694e816..daeb54c 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -18,17 +18,21 @@ #include #include +#include #define MSGPACK_MAX_STACK_SIZE (1024) #include "msgpack/unpack_define.h" using namespace std; -typedef struct unpack_user { - struct array_stack_type{unsigned int size, last;}; - array_stack_type array_stack[MSGPACK_MAX_STACK_SIZE]; - int array_current; - +struct array_context { + unsigned int size; + unsigned int last; + stack_item(unsigned int size) : size(size), last(0) + {} +}; +struct unpack_user { + stack array_stack; map str_cache; ~unpack_user() { @@ -38,7 +42,7 @@ typedef struct unpack_user { Py_DECREF(it->second); } } -} unpack_user; +}; #define msgpack_unpack_struct(name) \ @@ -60,7 +64,6 @@ typedef struct template_context template_context; static inline msgpack_unpack_object template_callback_root(unpack_user* u) { - u->array_current = -1; return NULL; } @@ -113,9 +116,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - int cur = ++u->array_current; - u->array_stack[cur].size = n; - u->array_stack[cur].last = 0; + u->array_stack.push(stack_item(n)); *o = PyList_New(n); } else { @@ -126,17 +127,13 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - int cur = u->array_current; - int n = u->array_stack[cur].size; - int last = u->array_stack[cur].last; + unsigned int n = u->array_stack.top().size; + unsigned int &last = u->array_stack.top().last; PyList_SetItem(*c, last, o); last++; if (last >= n) { - u->array_current--; - } - else { - u->array_stack[cur].last = last; + u->array_stack.pop(); } return 0; } From 46d7c656214073cde4c458c7cf1eb03e2d33dbd9 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 19:49:02 +0900 Subject: [PATCH 0169/1172] Fix compile error. --- python/msgpack/pack.h | 12 ------------ python/msgpack/unpack.h | 4 ++-- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index f3935fb..544950b 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -15,18 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#if _MSC_VER -typedef signed char uint8_t; -typedef unsigned char uint8_t; -typedef short int16_t; -typedef unsigned short uint16_t; -typedef int int32_t; -typedef unsigned int uint32_t; -typedef long long int64_t; -typedef unsigned long long uint64_t; -#else -#include -#endif #include #include diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index daeb54c..b753493 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -28,7 +28,7 @@ using namespace std; struct array_context { unsigned int size; unsigned int last; - stack_item(unsigned int size) : size(size), last(0) + array_context(unsigned int size) : size(size), last(0) {} }; struct unpack_user { @@ -116,7 +116,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - u->array_stack.push(stack_item(n)); + u->array_stack.push(array_context(n)); *o = PyList_New(n); } else { From 8e850dca4ae7481df612e12fd3bf6391f488decf Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 19:49:02 +0900 Subject: [PATCH 0170/1172] Fix compile error. --- msgpack/pack.h | 12 ------------ msgpack/unpack.h | 4 ++-- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/msgpack/pack.h b/msgpack/pack.h index f3935fb..544950b 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -15,18 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#if _MSC_VER -typedef signed char uint8_t; -typedef unsigned char uint8_t; -typedef short int16_t; -typedef unsigned short uint16_t; -typedef int int32_t; -typedef unsigned int uint32_t; -typedef long long int64_t; -typedef unsigned long long uint64_t; -#else -#include -#endif #include #include diff --git a/msgpack/unpack.h b/msgpack/unpack.h index daeb54c..b753493 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -28,7 +28,7 @@ using namespace std; struct array_context { unsigned int size; unsigned int last; - stack_item(unsigned int size) : size(size), last(0) + array_context(unsigned int size) : size(size), last(0) {} }; struct unpack_user { @@ -116,7 +116,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - u->array_stack.push(stack_item(n)); + u->array_stack.push(array_context(n)); *o = PyList_New(n); } else { From 7d5f04917ea4986db096ad647b36ce509583ae5c Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 19:55:46 +0900 Subject: [PATCH 0171/1172] Update manifest. --- python/MANIFEST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/MANIFEST b/python/MANIFEST index 8b21b4c..6983a33 100644 --- a/python/MANIFEST +++ b/python/MANIFEST @@ -1,7 +1,7 @@ -msgpack.cpp setup.py msgpack/pack.h msgpack/unpack.h +msgpack/_msgpack.pyx include/msgpack/pack_define.h include/msgpack/pack_template.h include/msgpack/unpack_define.h From d21e117a9228d09699bc08750fd66f093b8a27a3 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 19:55:46 +0900 Subject: [PATCH 0172/1172] Update manifest. --- MANIFEST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST b/MANIFEST index 8b21b4c..6983a33 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1,7 +1,7 @@ -msgpack.cpp setup.py msgpack/pack.h msgpack/unpack.h +msgpack/_msgpack.pyx include/msgpack/pack_define.h include/msgpack/pack_template.h include/msgpack/unpack_define.h From 5c0a89c258f5f875a2c8c38f8876991372db3260 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 22:50:05 +0900 Subject: [PATCH 0173/1172] Update manifest. --- MANIFEST | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST b/MANIFEST index 6983a33..1fe22ec 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2,6 +2,7 @@ setup.py msgpack/pack.h msgpack/unpack.h msgpack/_msgpack.pyx +msgpack/__init__.py include/msgpack/pack_define.h include/msgpack/pack_template.h include/msgpack/unpack_define.h From b8e5b918a3c76da04fc1e5653b94dd99bf0b83c6 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 22:50:05 +0900 Subject: [PATCH 0174/1172] Update manifest. --- python/MANIFEST | 1 + 1 file changed, 1 insertion(+) diff --git a/python/MANIFEST b/python/MANIFEST index 6983a33..1fe22ec 100644 --- a/python/MANIFEST +++ b/python/MANIFEST @@ -2,6 +2,7 @@ setup.py msgpack/pack.h msgpack/unpack.h msgpack/_msgpack.pyx +msgpack/__init__.py include/msgpack/pack_define.h include/msgpack/pack_template.h include/msgpack/unpack_define.h From df2acc856915cffbf932376c92fd6bdc58110a5b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:13:22 +0900 Subject: [PATCH 0175/1172] Fix setup script bug. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index eb897f2..1cbc24e 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ setup(name='msgpack', version=version, cmdclass={'build_ext': build_ext}, ext_modules=[msgpack_mod], + packages=['msgpack'], description=desc, long_description=long_desc, ) From 99d0a41ec6a6169456dcfe64cd6972520a768b71 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:13:22 +0900 Subject: [PATCH 0176/1172] Fix setup script bug. --- python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/setup.py b/python/setup.py index eb897f2..1cbc24e 100644 --- a/python/setup.py +++ b/python/setup.py @@ -32,6 +32,7 @@ setup(name='msgpack', version=version, cmdclass={'build_ext': build_ext}, ext_modules=[msgpack_mod], + packages=['msgpack'], description=desc, long_description=long_desc, ) From f2d07e5d6906567ce06d87dd21085828b3ebc1cc Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:13:39 +0900 Subject: [PATCH 0177/1172] Remove unneccessary value. --- msgpack/unpack.h | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index b753493..12702d8 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -25,14 +25,8 @@ using namespace std; -struct array_context { - unsigned int size; - unsigned int last; - array_context(unsigned int size) : size(size), last(0) - {} -}; struct unpack_user { - stack array_stack; + stack array_stack; map str_cache; ~unpack_user() { @@ -116,7 +110,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - u->array_stack.push(array_context(n)); + u->array_stack.push(0); *o = PyList_New(n); } else { @@ -127,12 +121,12 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - unsigned int n = u->array_stack.top().size; - unsigned int &last = u->array_stack.top().last; - - PyList_SetItem(*c, last, o); + unsigned int &last = u->array_stack.top(); + PyList_SET_ITEM(*c, last, o); last++; - if (last >= n) { + + Py_ssize_t len = PyList_GET_SIZE(*c); + if (last >= len) { u->array_stack.pop(); } return 0; From dd53b141ef6d2495d0cc14c8949180a74ae3caf6 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:13:39 +0900 Subject: [PATCH 0178/1172] Remove unneccessary value. --- python/msgpack/unpack.h | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index b753493..12702d8 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -25,14 +25,8 @@ using namespace std; -struct array_context { - unsigned int size; - unsigned int last; - array_context(unsigned int size) : size(size), last(0) - {} -}; struct unpack_user { - stack array_stack; + stack array_stack; map str_cache; ~unpack_user() { @@ -116,7 +110,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - u->array_stack.push(array_context(n)); + u->array_stack.push(0); *o = PyList_New(n); } else { @@ -127,12 +121,12 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - unsigned int n = u->array_stack.top().size; - unsigned int &last = u->array_stack.top().last; - - PyList_SetItem(*c, last, o); + unsigned int &last = u->array_stack.top(); + PyList_SET_ITEM(*c, last, o); last++; - if (last >= n) { + + Py_ssize_t len = PyList_GET_SIZE(*c); + if (last >= len) { u->array_stack.pop(); } return 0; From d32b48f98f09be8c829a8d48f8d7fcdb4bd0c61e Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:38:48 +0900 Subject: [PATCH 0179/1172] Remove duplicated values. --- include | 1 - msgpack/pack.h | 4 +- msgpack/pack_define.h | 26 ++ msgpack/pack_template.h | 741 ++++++++++++++++++++++++++++++++++++++ msgpack/unpack.h | 53 +-- msgpack/unpack_define.h | 129 +++++++ msgpack/unpack_template.h | 363 +++++++++++++++++++ 7 files changed, 1276 insertions(+), 41 deletions(-) delete mode 120000 include create mode 100644 msgpack/pack_define.h create mode 100644 msgpack/pack_template.h create mode 100644 msgpack/unpack_define.h create mode 100644 msgpack/unpack_template.h diff --git a/include b/include deleted file mode 120000 index a96aa0e..0000000 --- a/include +++ /dev/null @@ -1 +0,0 @@ -.. \ No newline at end of file diff --git a/msgpack/pack.h b/msgpack/pack.h index 544950b..9bd6b68 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -18,7 +18,7 @@ #include #include -#include "msgpack/pack_define.h" +#include "pack_define.h" #ifdef __cplusplus extern "C" { @@ -82,7 +82,7 @@ static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_ #define msgpack_pack_append_buffer(user, buf, len) \ return (*(user)->callback)((user)->data, (const char*)buf, len) -#include "msgpack/pack_template.h" +#include "pack_template.h" static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) { diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h new file mode 100644 index 0000000..33408e5 --- /dev/null +++ b/msgpack/pack_define.h @@ -0,0 +1,26 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_DEFINE_H__ +#define MSGPACK_PACK_DEFINE_H__ + +#include +#include +#include + +#endif /* msgpack/pack_define.h */ + diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h new file mode 100644 index 0000000..aa620f5 --- /dev/null +++ b/msgpack/pack_template.h @@ -0,0 +1,741 @@ +/* + * MessagePack packing routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + + +#ifdef __LITTLE_ENDIAN__ + +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] + + +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#elif __BIG_ENDIAN__ + +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] + + +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[1] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[3] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[7] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#endif + +#ifndef msgpack_pack_inline_func +#error msgpack_pack_inline_func template is not defined +#endif + +#ifndef msgpack_pack_user +#error msgpack_pack_user type is not defined +#endif + +#ifndef msgpack_pack_append_buffer +#error msgpack_pack_append_buffer callback is not defined +#endif + + +/* + * Integer + */ + +#define msgpack_pack_real_uint8(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ +} while(0) + +#define msgpack_pack_real_uint16(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + } else if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ +} while(0) + +#define msgpack_pack_real_uint32(x, d) \ +do { \ + if(d < (1<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_uint64(x, d) \ +do { \ + if(d < (1ULL<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1ULL<<16)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else if(d < (1ULL<<32)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int8(x, d) \ +do { \ + if(d < -(1<<5)) { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + } \ +} while(0) + +#define msgpack_pack_real_int16(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int32(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<15)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int64(x, d) \ +do { \ + if(d < -(1LL<<5)) { \ + if(d < -(1LL<<15)) { \ + if(d < -(1LL<<31)) { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } else { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } else { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + } else { \ + if(d < (1LL<<16)) { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } else { \ + if(d < (1LL<<32)) { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* unsigned 64 */ \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ + } \ +} while(0) + + +#ifdef msgpack_pack_inline_func_fastint + +msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) +{ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; + msgpack_pack_append_buffer(x, buf, 2); +} + +msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) +{ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) +{ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) +{ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) +{ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; + msgpack_pack_append_buffer(x, buf, 2); +} + +msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) +{ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) +{ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) +{ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +#undef msgpack_pack_inline_func_fastint +#endif + + +msgpack_pack_inline_func(_uint8)(msgpack_pack_user x, uint8_t d) +{ + msgpack_pack_real_uint8(x, d); +} + +msgpack_pack_inline_func(_uint16)(msgpack_pack_user x, uint16_t d) +{ + msgpack_pack_real_uint16(x, d); +} + +msgpack_pack_inline_func(_uint32)(msgpack_pack_user x, uint32_t d) +{ + msgpack_pack_real_uint32(x, d); +} + +msgpack_pack_inline_func(_uint64)(msgpack_pack_user x, uint64_t d) +{ + msgpack_pack_real_uint64(x, d); +} + +msgpack_pack_inline_func(_int8)(msgpack_pack_user x, int8_t d) +{ + msgpack_pack_real_int8(x, d); +} + +msgpack_pack_inline_func(_int16)(msgpack_pack_user x, int16_t d) +{ + msgpack_pack_real_int16(x, d); +} + +msgpack_pack_inline_func(_int32)(msgpack_pack_user x, int32_t d) +{ + msgpack_pack_real_int32(x, d); +} + +msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d) +{ + msgpack_pack_real_int64(x, d); +} + + +#ifdef msgpack_pack_inline_func_cint + +msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d) +{ +#if defined(SIZEOF_SHORT) || defined(SHRT_MAX) +#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(short) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(short) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d) +{ +#if defined(SIZEOF_INT) || defined(INT_MAX) +#if SIZEOF_INT == 2 || INT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(int) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(int) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d) +{ +#if defined(SIZEOF_LONG) || defined(LONG_MAX) +#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d) +{ +#if defined(SIZEOF_SHORT) || defined(USHRT_MAX) +#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned short) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned short) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d) +{ +#if defined(SIZEOF_INT) || defined(UINT_MAX) +#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d) +{ +#if defined(SIZEOF_LONG) || defined(ULONG_MAX) +#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned long long) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned long long) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +#undef msgpack_pack_inline_func_cint +#endif + + + +/* + * Float + */ + +msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) +{ + union { char buf[4]; uint32_t num; } f; + *((float*)&f.buf) = d; // FIXME + const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) +{ + union { char buf[8]; uint64_t num; } f; + *((double*)&f.buf) = d; // FIXME + const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; + msgpack_pack_append_buffer(x, buf, 9); +} + + +/* + * Nil + */ + +msgpack_pack_inline_func(_nil)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc0; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Boolean + */ + +msgpack_pack_inline_func(_true)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc3; + msgpack_pack_append_buffer(x, &d, 1); +} + +msgpack_pack_inline_func(_false)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc2; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Array + */ + +msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x90 | n; + msgpack_pack_append_buffer(x, &d, 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Map + */ + +msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x80 | n; + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xde, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Raw + */ + +msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) +{ + if(l < 32) { + unsigned char d = 0xa0 | l; + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + } else if(l < 65536) { + uint16_t d = (uint16_t)l; + unsigned char buf[3] = {0xda, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)l; + unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + +msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l) +{ + msgpack_pack_append_buffer(x, (const unsigned char*)b, l); +} + +#undef msgpack_pack_inline_func +#undef msgpack_pack_user +#undef msgpack_pack_append_buffer + +#undef STORE8_BE8 + +#undef STORE16_BE8 +#undef STORE16_BE16 + +#undef STORE32_BE8 +#undef STORE32_BE16 +#undef STORE32_BE32 + +#undef STORE64_BE8 +#undef STORE64_BE16 +#undef STORE64_BE32 +#undef STORE64_BE64 + +#undef msgpack_pack_real_uint8 +#undef msgpack_pack_real_uint16 +#undef msgpack_pack_real_uint32 +#undef msgpack_pack_real_uint64 +#undef msgpack_pack_real_int8 +#undef msgpack_pack_real_int16 +#undef msgpack_pack_real_int32 +#undef msgpack_pack_real_int64 + diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 12702d8..cc48d9c 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -21,18 +21,18 @@ #include #define MSGPACK_MAX_STACK_SIZE (1024) -#include "msgpack/unpack_define.h" +#include "unpack_define.h" using namespace std; +typedef map str_cach_t; struct unpack_user { - stack array_stack; - map str_cache; + str_cach_t strcache; ~unpack_user() { - map::iterator it, itend; - itend = str_cache.end(); - for (it = str_cache.begin(); it != itend; ++it) { + str_cach_t::iterator it, itend; + itend = strcache.end(); + for (it = strcache.begin(); it != itend; ++it) { Py_DECREF(it->second); } } @@ -108,35 +108,13 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* { Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ - if (n > 0) { - u->array_stack.push(0); - *o = PyList_New(n); - } - else { - *o = PyList_New(0); - } - return 0; -} +{ *o = PyList_New(n); return 0; } -static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) -{ - unsigned int &last = u->array_stack.top(); - PyList_SET_ITEM(*c, last, o); - last++; - - Py_ssize_t len = PyList_GET_SIZE(*c); - if (last >= len) { - u->array_stack.pop(); - } - return 0; -} +static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) +{ PyList_SET_ITEM(*c, current, o); return 0; } static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ - *o = PyDict_New(); - return 0; -} +{ *o = PyDict_New(); return 0; } static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { @@ -150,16 +128,15 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha { if (l < 16) { string s(p, l); - map::iterator it = u->str_cache.find(s); - if (it != u->str_cache.end()) { + str_cach_t ::iterator it = u->strcache.find(s); + if (it != u->strcache.end()) { *o = it->second; - Py_INCREF(*o); } else { *o = PyString_FromStringAndSize(p, l); - Py_INCREF(*o); - u->str_cache[s] = *o; + u->strcache[s] = *o; } + Py_INCREF(*o); } else { *o = PyString_FromStringAndSize(p, l); @@ -167,4 +144,4 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha return 0; } -#include "msgpack/unpack_template.h" +#include "unpack_template.h" diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h new file mode 100644 index 0000000..63668c2 --- /dev/null +++ b/msgpack/unpack_define.h @@ -0,0 +1,129 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_UNPACK_DEFINE_H__ +#define MSGPACK_UNPACK_DEFINE_H__ + +#include +#include +#include +#include +#include +#ifndef __WIN32__ +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifndef MSGPACK_MAX_STACK_SIZE +#define MSGPACK_MAX_STACK_SIZE 16 +#endif + + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#define msgpack_betoh16(x) ntohs(x) +#define msgpack_betoh32(x) ntohl(x) + +#ifdef __LITTLE_ENDIAN__ +#if defined(__bswap_64) +# define msgpack_betoh64(x) __bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) +#else +static inline uint64_t msgpack_betoh64(uint64_t x) { + return ((x << 56) & 0xff00000000000000ULL ) | + ((x << 40) & 0x00ff000000000000ULL ) | + ((x << 24) & 0x0000ff0000000000ULL ) | + ((x << 8) & 0x000000ff00000000ULL ) | + ((x >> 8) & 0x00000000ff000000ULL ) | + ((x >> 24) & 0x0000000000ff0000ULL ) | + ((x >> 40) & 0x000000000000ff00ULL ) | + ((x >> 56) & 0x00000000000000ffULL ) ; +} +#endif +#else +#define msgpack_betoh64(x) (x) +#endif + + +typedef enum { + CS_HEADER = 0x00, // nil + + //CS_ = 0x01, + //CS_ = 0x02, // false + //CS_ = 0x03, // true + + //CS_ = 0x04, + //CS_ = 0x05, + //CS_ = 0x06, + //CS_ = 0x07, + + //CS_ = 0x08, + //CS_ = 0x09, + CS_FLOAT = 0x0a, + CS_DOUBLE = 0x0b, + CS_UINT_8 = 0x0c, + CS_UINT_16 = 0x0d, + CS_UINT_32 = 0x0e, + CS_UINT_64 = 0x0f, + CS_INT_8 = 0x10, + CS_INT_16 = 0x11, + CS_INT_32 = 0x12, + CS_INT_64 = 0x13, + + //CS_ = 0x14, + //CS_ = 0x15, + //CS_BIG_INT_16 = 0x16, + //CS_BIG_INT_32 = 0x17, + //CS_BIG_FLOAT_16 = 0x18, + //CS_BIG_FLOAT_32 = 0x19, + CS_RAW_16 = 0x1a, + CS_RAW_32 = 0x1b, + CS_ARRAY_16 = 0x1c, + CS_ARRAY_32 = 0x1d, + CS_MAP_16 = 0x1e, + CS_MAP_32 = 0x1f, + + //ACS_BIG_INT_VALUE, + //ACS_BIG_FLOAT_VALUE, + ACS_RAW_VALUE, +} msgpack_unpack_state; + + +typedef enum { + CT_ARRAY_ITEM, + CT_MAP_KEY, + CT_MAP_VALUE, +} msgpack_container_type; + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/unpack_define.h */ + diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h new file mode 100644 index 0000000..db33368 --- /dev/null +++ b/msgpack/unpack_template.h @@ -0,0 +1,363 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef msgpack_unpack_func +#error msgpack_unpack_func template is not defined +#endif + +#ifndef msgpack_unpack_callback +#error msgpack_unpack_callback template is not defined +#endif + +#ifndef msgpack_unpack_struct +#error msgpack_unpack_struct template is not defined +#endif + +#ifndef msgpack_unpack_struct_decl +#define msgpack_unpack_struct_decl(name) msgpack_unpack_struct(name) +#endif + +#ifndef msgpack_unpack_object +#error msgpack_unpack_object type is not defined +#endif + +#ifndef msgpack_unpack_user +#error msgpack_unpack_user type is not defined +#endif + + +msgpack_unpack_struct_decl(_stack) { + msgpack_unpack_object obj; + size_t curr; + size_t count; + unsigned int ct; + msgpack_unpack_object map_key; +}; + +msgpack_unpack_struct_decl(_context) { + msgpack_unpack_user user; + unsigned int cs; + unsigned int trail; + unsigned int top; + msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE]; +}; + + +msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) +{ + ctx->cs = CS_HEADER; + ctx->trail = 0; + ctx->top = 0; + ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); +} + +msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) +{ + return (ctx)->stack[0].obj; +} + + +msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) +{ + assert(len >= *off); + + const unsigned char* p = (unsigned char*)data + *off; + const unsigned char* const pe = (unsigned char*)data + len; + const void* n = NULL; + + unsigned int trail = ctx->trail; + unsigned int cs = ctx->cs; + unsigned int top = ctx->top; + msgpack_unpack_struct(_stack)* stack = ctx->stack; + msgpack_unpack_user* user = &ctx->user; + + msgpack_unpack_object obj; + msgpack_unpack_struct(_stack)* c = NULL; + + int ret; + +#define push_simple_value(func) \ + if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \ + goto _push +#define push_fixed_value(func, arg) \ + if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \ + goto _push +#define push_variable_value(func, base, pos, len) \ + if(msgpack_unpack_callback(func)(user, \ + (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ + goto _push + +#define again_fixed_trail(_cs, trail_len) \ + trail = trail_len; \ + cs = _cs; \ + goto _fixed_trail_again +#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ + trail = trail_len; \ + if(trail == 0) { goto ifzero; } \ + cs = _cs; \ + goto _fixed_trail_again + +#define start_container(func, count_, ct_) \ + if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ + if((count_) == 0) { obj = stack[top].obj; goto _push; } \ + if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ + stack[top].ct = ct_; \ + stack[top].curr = 0; \ + stack[top].count = count_; \ + /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ + /*printf("stack push %d\n", top);*/ \ + ++top; \ + goto _header_again + +#define NEXT_CS(p) \ + ((unsigned int)*p & 0x1f) + +#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) +#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) + + if(p == pe) { goto _out; } + do { + switch(cs) { + case CS_HEADER: + switch(*p) { + case 0x00 ... 0x7f: // Positive Fixnum + push_fixed_value(_uint8, *(uint8_t*)p); + case 0xe0 ... 0xff: // Negative Fixnum + push_fixed_value(_int8, *(int8_t*)p); + case 0xc0 ... 0xdf: // Variable + switch(*p) { + case 0xc0: // nil + push_simple_value(_nil); + //case 0xc1: // string + // again_terminal_trail(NEXT_CS(p), p+1); + case 0xc2: // false + push_simple_value(_false); + case 0xc3: // true + push_simple_value(_true); + //case 0xc4: + //case 0xc5: + //case 0xc6: + //case 0xc7: + //case 0xc8: + //case 0xc9: + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); + //case 0xd4: + //case 0xd5: + //case 0xd6: // big integer 16 + //case 0xd7: // big integer 32 + //case 0xd8: // big float 16 + //case 0xd9: // big float 32 + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); + default: + goto _failed; + } + case 0xa0 ... 0xbf: // FixRaw + again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); + case 0x90 ... 0x9f: // FixArray + start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); + case 0x80 ... 0x8f: // FixMap + start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); + + default: + goto _failed; + } + // end CS_HEADER + + + _fixed_trail_again: + ++p; + + default: + if((size_t)(pe - p) < trail) { goto _out; } + n = p; p += trail - 1; + switch(cs) { + //case CS_ + //case CS_ + case CS_FLOAT: { + union { uint32_t num; char buf[4]; } f; + f.num = PTR_CAST_32(n); // FIXME + push_fixed_value(_float, *((float*)f.buf)); } + case CS_DOUBLE: { + union { uint64_t num; char buf[8]; } f; + f.num = PTR_CAST_64(n); // FIXME + push_fixed_value(_double, *((double*)f.buf)); } + case CS_UINT_8: + push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n)); + case CS_UINT_16: + push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n)); + case CS_UINT_32: + push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n)); + case CS_UINT_64: + push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n)); + + case CS_INT_8: + push_fixed_value(_int8, (int8_t)PTR_CAST_8(n)); + case CS_INT_16: + push_fixed_value(_int16, (int16_t)PTR_CAST_16(n)); + case CS_INT_32: + push_fixed_value(_int32, (int32_t)PTR_CAST_32(n)); + case CS_INT_64: + push_fixed_value(_int64, (int64_t)PTR_CAST_64(n)); + + //case CS_ + //case CS_ + //case CS_BIG_INT_16: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); + //case CS_BIG_INT_32: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); + //case ACS_BIG_INT_VALUE: + //_big_int_zero: + // // FIXME + // push_variable_value(_big_int, data, n, trail); + + //case CS_BIG_FLOAT_16: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); + //case CS_BIG_FLOAT_32: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); + //case ACS_BIG_FLOAT_VALUE: + //_big_float_zero: + // // FIXME + // push_variable_value(_big_float, data, n, trail); + + case CS_RAW_16: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); + case CS_RAW_32: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); + case ACS_RAW_VALUE: + _raw_zero: + push_variable_value(_raw, data, n, trail); + + case CS_ARRAY_16: + start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + case CS_ARRAY_32: + /* FIXME security guard */ + start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + + case CS_MAP_16: + start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + case CS_MAP_32: + /* FIXME security guard */ + start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + + default: + goto _failed; + } + } + +_push: + if(top == 0) { goto _finish; } + c = &stack[top-1]; + switch(c->ct) { + case CT_ARRAY_ITEM: + if(msgpack_unpack_callback(_array_item)(user, c->curr, &c->obj, obj) < 0) { goto _failed; } + if(++c->curr == c->count) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + goto _header_again; + case CT_MAP_KEY: + c->map_key = obj; + c->ct = CT_MAP_VALUE; + goto _header_again; + case CT_MAP_VALUE: + if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + c->ct = CT_MAP_KEY; + goto _header_again; + + default: + goto _failed; + } + +_header_again: + cs = CS_HEADER; + ++p; + } while(p != pe); + goto _out; + + +_finish: + stack[0].obj = obj; + ++p; + ret = 1; + /*printf("-- finish --\n"); */ + goto _end; + +_failed: + /*printf("** FAILED **\n"); */ + ret = -1; + goto _end; + +_out: + ret = 0; + goto _end; + +_end: + ctx->cs = cs; + ctx->trail = trail; + ctx->top = top; + *off = p - (const unsigned char*)data; + + return ret; +} + + +#undef msgpack_unpack_func +#undef msgpack_unpack_callback +#undef msgpack_unpack_struct +#undef msgpack_unpack_object +#undef msgpack_unpack_user + +#undef push_simple_value +#undef push_fixed_value +#undef push_variable_value +#undef again_fixed_trail +#undef again_fixed_trail_if_zero +#undef start_container + +#undef NEXT_CS +#undef PTR_CAST_8 +#undef PTR_CAST_16 +#undef PTR_CAST_32 +#undef PTR_CAST_64 + From 3fd28d07928c51cb563f35d9dddb0c7867ac9875 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:38:48 +0900 Subject: [PATCH 0180/1172] Remove duplicated values. --- python/include | 1 - python/msgpack/pack.h | 4 +- python/msgpack/pack_define.h | 26 ++ python/msgpack/pack_template.h | 741 +++++++++++++++++++++++++++++++ python/msgpack/unpack.h | 53 +-- python/msgpack/unpack_define.h | 129 ++++++ python/msgpack/unpack_template.h | 363 +++++++++++++++ 7 files changed, 1276 insertions(+), 41 deletions(-) delete mode 120000 python/include create mode 100644 python/msgpack/pack_define.h create mode 100644 python/msgpack/pack_template.h create mode 100644 python/msgpack/unpack_define.h create mode 100644 python/msgpack/unpack_template.h diff --git a/python/include b/python/include deleted file mode 120000 index a96aa0e..0000000 --- a/python/include +++ /dev/null @@ -1 +0,0 @@ -.. \ No newline at end of file diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index 544950b..9bd6b68 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -18,7 +18,7 @@ #include #include -#include "msgpack/pack_define.h" +#include "pack_define.h" #ifdef __cplusplus extern "C" { @@ -82,7 +82,7 @@ static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_ #define msgpack_pack_append_buffer(user, buf, len) \ return (*(user)->callback)((user)->data, (const char*)buf, len) -#include "msgpack/pack_template.h" +#include "pack_template.h" static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) { diff --git a/python/msgpack/pack_define.h b/python/msgpack/pack_define.h new file mode 100644 index 0000000..33408e5 --- /dev/null +++ b/python/msgpack/pack_define.h @@ -0,0 +1,26 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_DEFINE_H__ +#define MSGPACK_PACK_DEFINE_H__ + +#include +#include +#include + +#endif /* msgpack/pack_define.h */ + diff --git a/python/msgpack/pack_template.h b/python/msgpack/pack_template.h new file mode 100644 index 0000000..aa620f5 --- /dev/null +++ b/python/msgpack/pack_template.h @@ -0,0 +1,741 @@ +/* + * MessagePack packing routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + + +#ifdef __LITTLE_ENDIAN__ + +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] + + +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#elif __BIG_ENDIAN__ + +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] + + +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[1] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[3] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[7] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#endif + +#ifndef msgpack_pack_inline_func +#error msgpack_pack_inline_func template is not defined +#endif + +#ifndef msgpack_pack_user +#error msgpack_pack_user type is not defined +#endif + +#ifndef msgpack_pack_append_buffer +#error msgpack_pack_append_buffer callback is not defined +#endif + + +/* + * Integer + */ + +#define msgpack_pack_real_uint8(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ +} while(0) + +#define msgpack_pack_real_uint16(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + } else if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ +} while(0) + +#define msgpack_pack_real_uint32(x, d) \ +do { \ + if(d < (1<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_uint64(x, d) \ +do { \ + if(d < (1ULL<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1ULL<<16)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else if(d < (1ULL<<32)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int8(x, d) \ +do { \ + if(d < -(1<<5)) { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + } \ +} while(0) + +#define msgpack_pack_real_int16(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int32(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<15)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int64(x, d) \ +do { \ + if(d < -(1LL<<5)) { \ + if(d < -(1LL<<15)) { \ + if(d < -(1LL<<31)) { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } else { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } else { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + } else { \ + if(d < (1LL<<16)) { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } else { \ + if(d < (1LL<<32)) { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* unsigned 64 */ \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ + } \ +} while(0) + + +#ifdef msgpack_pack_inline_func_fastint + +msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) +{ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; + msgpack_pack_append_buffer(x, buf, 2); +} + +msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) +{ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) +{ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) +{ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) +{ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; + msgpack_pack_append_buffer(x, buf, 2); +} + +msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) +{ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) +{ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) +{ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +#undef msgpack_pack_inline_func_fastint +#endif + + +msgpack_pack_inline_func(_uint8)(msgpack_pack_user x, uint8_t d) +{ + msgpack_pack_real_uint8(x, d); +} + +msgpack_pack_inline_func(_uint16)(msgpack_pack_user x, uint16_t d) +{ + msgpack_pack_real_uint16(x, d); +} + +msgpack_pack_inline_func(_uint32)(msgpack_pack_user x, uint32_t d) +{ + msgpack_pack_real_uint32(x, d); +} + +msgpack_pack_inline_func(_uint64)(msgpack_pack_user x, uint64_t d) +{ + msgpack_pack_real_uint64(x, d); +} + +msgpack_pack_inline_func(_int8)(msgpack_pack_user x, int8_t d) +{ + msgpack_pack_real_int8(x, d); +} + +msgpack_pack_inline_func(_int16)(msgpack_pack_user x, int16_t d) +{ + msgpack_pack_real_int16(x, d); +} + +msgpack_pack_inline_func(_int32)(msgpack_pack_user x, int32_t d) +{ + msgpack_pack_real_int32(x, d); +} + +msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d) +{ + msgpack_pack_real_int64(x, d); +} + + +#ifdef msgpack_pack_inline_func_cint + +msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d) +{ +#if defined(SIZEOF_SHORT) || defined(SHRT_MAX) +#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(short) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(short) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d) +{ +#if defined(SIZEOF_INT) || defined(INT_MAX) +#if SIZEOF_INT == 2 || INT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(int) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(int) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d) +{ +#if defined(SIZEOF_LONG) || defined(LONG_MAX) +#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d) +{ +#if defined(SIZEOF_SHORT) || defined(USHRT_MAX) +#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned short) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned short) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d) +{ +#if defined(SIZEOF_INT) || defined(UINT_MAX) +#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d) +{ +#if defined(SIZEOF_LONG) || defined(ULONG_MAX) +#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned long long) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned long long) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +#undef msgpack_pack_inline_func_cint +#endif + + + +/* + * Float + */ + +msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) +{ + union { char buf[4]; uint32_t num; } f; + *((float*)&f.buf) = d; // FIXME + const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) +{ + union { char buf[8]; uint64_t num; } f; + *((double*)&f.buf) = d; // FIXME + const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; + msgpack_pack_append_buffer(x, buf, 9); +} + + +/* + * Nil + */ + +msgpack_pack_inline_func(_nil)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc0; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Boolean + */ + +msgpack_pack_inline_func(_true)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc3; + msgpack_pack_append_buffer(x, &d, 1); +} + +msgpack_pack_inline_func(_false)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc2; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Array + */ + +msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x90 | n; + msgpack_pack_append_buffer(x, &d, 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Map + */ + +msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x80 | n; + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xde, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Raw + */ + +msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) +{ + if(l < 32) { + unsigned char d = 0xa0 | l; + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + } else if(l < 65536) { + uint16_t d = (uint16_t)l; + unsigned char buf[3] = {0xda, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)l; + unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + +msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l) +{ + msgpack_pack_append_buffer(x, (const unsigned char*)b, l); +} + +#undef msgpack_pack_inline_func +#undef msgpack_pack_user +#undef msgpack_pack_append_buffer + +#undef STORE8_BE8 + +#undef STORE16_BE8 +#undef STORE16_BE16 + +#undef STORE32_BE8 +#undef STORE32_BE16 +#undef STORE32_BE32 + +#undef STORE64_BE8 +#undef STORE64_BE16 +#undef STORE64_BE32 +#undef STORE64_BE64 + +#undef msgpack_pack_real_uint8 +#undef msgpack_pack_real_uint16 +#undef msgpack_pack_real_uint32 +#undef msgpack_pack_real_uint64 +#undef msgpack_pack_real_int8 +#undef msgpack_pack_real_int16 +#undef msgpack_pack_real_int32 +#undef msgpack_pack_real_int64 + diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 12702d8..cc48d9c 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -21,18 +21,18 @@ #include #define MSGPACK_MAX_STACK_SIZE (1024) -#include "msgpack/unpack_define.h" +#include "unpack_define.h" using namespace std; +typedef map str_cach_t; struct unpack_user { - stack array_stack; - map str_cache; + str_cach_t strcache; ~unpack_user() { - map::iterator it, itend; - itend = str_cache.end(); - for (it = str_cache.begin(); it != itend; ++it) { + str_cach_t::iterator it, itend; + itend = strcache.end(); + for (it = strcache.begin(); it != itend; ++it) { Py_DECREF(it->second); } } @@ -108,35 +108,13 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* { Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ - if (n > 0) { - u->array_stack.push(0); - *o = PyList_New(n); - } - else { - *o = PyList_New(0); - } - return 0; -} +{ *o = PyList_New(n); return 0; } -static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) -{ - unsigned int &last = u->array_stack.top(); - PyList_SET_ITEM(*c, last, o); - last++; - - Py_ssize_t len = PyList_GET_SIZE(*c); - if (last >= len) { - u->array_stack.pop(); - } - return 0; -} +static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) +{ PyList_SET_ITEM(*c, current, o); return 0; } static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ - *o = PyDict_New(); - return 0; -} +{ *o = PyDict_New(); return 0; } static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { @@ -150,16 +128,15 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha { if (l < 16) { string s(p, l); - map::iterator it = u->str_cache.find(s); - if (it != u->str_cache.end()) { + str_cach_t ::iterator it = u->strcache.find(s); + if (it != u->strcache.end()) { *o = it->second; - Py_INCREF(*o); } else { *o = PyString_FromStringAndSize(p, l); - Py_INCREF(*o); - u->str_cache[s] = *o; + u->strcache[s] = *o; } + Py_INCREF(*o); } else { *o = PyString_FromStringAndSize(p, l); @@ -167,4 +144,4 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha return 0; } -#include "msgpack/unpack_template.h" +#include "unpack_template.h" diff --git a/python/msgpack/unpack_define.h b/python/msgpack/unpack_define.h new file mode 100644 index 0000000..63668c2 --- /dev/null +++ b/python/msgpack/unpack_define.h @@ -0,0 +1,129 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_UNPACK_DEFINE_H__ +#define MSGPACK_UNPACK_DEFINE_H__ + +#include +#include +#include +#include +#include +#ifndef __WIN32__ +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifndef MSGPACK_MAX_STACK_SIZE +#define MSGPACK_MAX_STACK_SIZE 16 +#endif + + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#define msgpack_betoh16(x) ntohs(x) +#define msgpack_betoh32(x) ntohl(x) + +#ifdef __LITTLE_ENDIAN__ +#if defined(__bswap_64) +# define msgpack_betoh64(x) __bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) +#else +static inline uint64_t msgpack_betoh64(uint64_t x) { + return ((x << 56) & 0xff00000000000000ULL ) | + ((x << 40) & 0x00ff000000000000ULL ) | + ((x << 24) & 0x0000ff0000000000ULL ) | + ((x << 8) & 0x000000ff00000000ULL ) | + ((x >> 8) & 0x00000000ff000000ULL ) | + ((x >> 24) & 0x0000000000ff0000ULL ) | + ((x >> 40) & 0x000000000000ff00ULL ) | + ((x >> 56) & 0x00000000000000ffULL ) ; +} +#endif +#else +#define msgpack_betoh64(x) (x) +#endif + + +typedef enum { + CS_HEADER = 0x00, // nil + + //CS_ = 0x01, + //CS_ = 0x02, // false + //CS_ = 0x03, // true + + //CS_ = 0x04, + //CS_ = 0x05, + //CS_ = 0x06, + //CS_ = 0x07, + + //CS_ = 0x08, + //CS_ = 0x09, + CS_FLOAT = 0x0a, + CS_DOUBLE = 0x0b, + CS_UINT_8 = 0x0c, + CS_UINT_16 = 0x0d, + CS_UINT_32 = 0x0e, + CS_UINT_64 = 0x0f, + CS_INT_8 = 0x10, + CS_INT_16 = 0x11, + CS_INT_32 = 0x12, + CS_INT_64 = 0x13, + + //CS_ = 0x14, + //CS_ = 0x15, + //CS_BIG_INT_16 = 0x16, + //CS_BIG_INT_32 = 0x17, + //CS_BIG_FLOAT_16 = 0x18, + //CS_BIG_FLOAT_32 = 0x19, + CS_RAW_16 = 0x1a, + CS_RAW_32 = 0x1b, + CS_ARRAY_16 = 0x1c, + CS_ARRAY_32 = 0x1d, + CS_MAP_16 = 0x1e, + CS_MAP_32 = 0x1f, + + //ACS_BIG_INT_VALUE, + //ACS_BIG_FLOAT_VALUE, + ACS_RAW_VALUE, +} msgpack_unpack_state; + + +typedef enum { + CT_ARRAY_ITEM, + CT_MAP_KEY, + CT_MAP_VALUE, +} msgpack_container_type; + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/unpack_define.h */ + diff --git a/python/msgpack/unpack_template.h b/python/msgpack/unpack_template.h new file mode 100644 index 0000000..db33368 --- /dev/null +++ b/python/msgpack/unpack_template.h @@ -0,0 +1,363 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef msgpack_unpack_func +#error msgpack_unpack_func template is not defined +#endif + +#ifndef msgpack_unpack_callback +#error msgpack_unpack_callback template is not defined +#endif + +#ifndef msgpack_unpack_struct +#error msgpack_unpack_struct template is not defined +#endif + +#ifndef msgpack_unpack_struct_decl +#define msgpack_unpack_struct_decl(name) msgpack_unpack_struct(name) +#endif + +#ifndef msgpack_unpack_object +#error msgpack_unpack_object type is not defined +#endif + +#ifndef msgpack_unpack_user +#error msgpack_unpack_user type is not defined +#endif + + +msgpack_unpack_struct_decl(_stack) { + msgpack_unpack_object obj; + size_t curr; + size_t count; + unsigned int ct; + msgpack_unpack_object map_key; +}; + +msgpack_unpack_struct_decl(_context) { + msgpack_unpack_user user; + unsigned int cs; + unsigned int trail; + unsigned int top; + msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE]; +}; + + +msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) +{ + ctx->cs = CS_HEADER; + ctx->trail = 0; + ctx->top = 0; + ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); +} + +msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) +{ + return (ctx)->stack[0].obj; +} + + +msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) +{ + assert(len >= *off); + + const unsigned char* p = (unsigned char*)data + *off; + const unsigned char* const pe = (unsigned char*)data + len; + const void* n = NULL; + + unsigned int trail = ctx->trail; + unsigned int cs = ctx->cs; + unsigned int top = ctx->top; + msgpack_unpack_struct(_stack)* stack = ctx->stack; + msgpack_unpack_user* user = &ctx->user; + + msgpack_unpack_object obj; + msgpack_unpack_struct(_stack)* c = NULL; + + int ret; + +#define push_simple_value(func) \ + if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \ + goto _push +#define push_fixed_value(func, arg) \ + if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \ + goto _push +#define push_variable_value(func, base, pos, len) \ + if(msgpack_unpack_callback(func)(user, \ + (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ + goto _push + +#define again_fixed_trail(_cs, trail_len) \ + trail = trail_len; \ + cs = _cs; \ + goto _fixed_trail_again +#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ + trail = trail_len; \ + if(trail == 0) { goto ifzero; } \ + cs = _cs; \ + goto _fixed_trail_again + +#define start_container(func, count_, ct_) \ + if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ + if((count_) == 0) { obj = stack[top].obj; goto _push; } \ + if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ + stack[top].ct = ct_; \ + stack[top].curr = 0; \ + stack[top].count = count_; \ + /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ + /*printf("stack push %d\n", top);*/ \ + ++top; \ + goto _header_again + +#define NEXT_CS(p) \ + ((unsigned int)*p & 0x1f) + +#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) +#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) + + if(p == pe) { goto _out; } + do { + switch(cs) { + case CS_HEADER: + switch(*p) { + case 0x00 ... 0x7f: // Positive Fixnum + push_fixed_value(_uint8, *(uint8_t*)p); + case 0xe0 ... 0xff: // Negative Fixnum + push_fixed_value(_int8, *(int8_t*)p); + case 0xc0 ... 0xdf: // Variable + switch(*p) { + case 0xc0: // nil + push_simple_value(_nil); + //case 0xc1: // string + // again_terminal_trail(NEXT_CS(p), p+1); + case 0xc2: // false + push_simple_value(_false); + case 0xc3: // true + push_simple_value(_true); + //case 0xc4: + //case 0xc5: + //case 0xc6: + //case 0xc7: + //case 0xc8: + //case 0xc9: + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); + //case 0xd4: + //case 0xd5: + //case 0xd6: // big integer 16 + //case 0xd7: // big integer 32 + //case 0xd8: // big float 16 + //case 0xd9: // big float 32 + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); + default: + goto _failed; + } + case 0xa0 ... 0xbf: // FixRaw + again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); + case 0x90 ... 0x9f: // FixArray + start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); + case 0x80 ... 0x8f: // FixMap + start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); + + default: + goto _failed; + } + // end CS_HEADER + + + _fixed_trail_again: + ++p; + + default: + if((size_t)(pe - p) < trail) { goto _out; } + n = p; p += trail - 1; + switch(cs) { + //case CS_ + //case CS_ + case CS_FLOAT: { + union { uint32_t num; char buf[4]; } f; + f.num = PTR_CAST_32(n); // FIXME + push_fixed_value(_float, *((float*)f.buf)); } + case CS_DOUBLE: { + union { uint64_t num; char buf[8]; } f; + f.num = PTR_CAST_64(n); // FIXME + push_fixed_value(_double, *((double*)f.buf)); } + case CS_UINT_8: + push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n)); + case CS_UINT_16: + push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n)); + case CS_UINT_32: + push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n)); + case CS_UINT_64: + push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n)); + + case CS_INT_8: + push_fixed_value(_int8, (int8_t)PTR_CAST_8(n)); + case CS_INT_16: + push_fixed_value(_int16, (int16_t)PTR_CAST_16(n)); + case CS_INT_32: + push_fixed_value(_int32, (int32_t)PTR_CAST_32(n)); + case CS_INT_64: + push_fixed_value(_int64, (int64_t)PTR_CAST_64(n)); + + //case CS_ + //case CS_ + //case CS_BIG_INT_16: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); + //case CS_BIG_INT_32: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); + //case ACS_BIG_INT_VALUE: + //_big_int_zero: + // // FIXME + // push_variable_value(_big_int, data, n, trail); + + //case CS_BIG_FLOAT_16: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); + //case CS_BIG_FLOAT_32: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); + //case ACS_BIG_FLOAT_VALUE: + //_big_float_zero: + // // FIXME + // push_variable_value(_big_float, data, n, trail); + + case CS_RAW_16: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); + case CS_RAW_32: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); + case ACS_RAW_VALUE: + _raw_zero: + push_variable_value(_raw, data, n, trail); + + case CS_ARRAY_16: + start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + case CS_ARRAY_32: + /* FIXME security guard */ + start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + + case CS_MAP_16: + start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + case CS_MAP_32: + /* FIXME security guard */ + start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + + default: + goto _failed; + } + } + +_push: + if(top == 0) { goto _finish; } + c = &stack[top-1]; + switch(c->ct) { + case CT_ARRAY_ITEM: + if(msgpack_unpack_callback(_array_item)(user, c->curr, &c->obj, obj) < 0) { goto _failed; } + if(++c->curr == c->count) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + goto _header_again; + case CT_MAP_KEY: + c->map_key = obj; + c->ct = CT_MAP_VALUE; + goto _header_again; + case CT_MAP_VALUE: + if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + c->ct = CT_MAP_KEY; + goto _header_again; + + default: + goto _failed; + } + +_header_again: + cs = CS_HEADER; + ++p; + } while(p != pe); + goto _out; + + +_finish: + stack[0].obj = obj; + ++p; + ret = 1; + /*printf("-- finish --\n"); */ + goto _end; + +_failed: + /*printf("** FAILED **\n"); */ + ret = -1; + goto _end; + +_out: + ret = 0; + goto _end; + +_end: + ctx->cs = cs; + ctx->trail = trail; + ctx->top = top; + *off = p - (const unsigned char*)data; + + return ret; +} + + +#undef msgpack_unpack_func +#undef msgpack_unpack_callback +#undef msgpack_unpack_struct +#undef msgpack_unpack_object +#undef msgpack_unpack_user + +#undef push_simple_value +#undef push_fixed_value +#undef push_variable_value +#undef again_fixed_trail +#undef again_fixed_trail_if_zero +#undef start_container + +#undef NEXT_CS +#undef PTR_CAST_8 +#undef PTR_CAST_16 +#undef PTR_CAST_32 +#undef PTR_CAST_64 + From de6afa5285c2f748d0c0c72b930fce77f68fcb85 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:54:47 +0900 Subject: [PATCH 0181/1172] Reduce memory footprint. --- msgpack/unpack.h | 6 ++---- msgpack/unpack_template.h | 7 +++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index cc48d9c..12afb99 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -23,9 +23,7 @@ #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -using namespace std; - -typedef map str_cach_t; +typedef std::map str_cach_t; struct unpack_user { str_cach_t strcache; @@ -127,7 +125,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { if (l < 16) { - string s(p, l); + std::string s(p, l); str_cach_t ::iterator it = u->strcache.find(s); if (it != u->strcache.end()) { *o = it->second; diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index db33368..c960c3a 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -43,10 +43,13 @@ msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; - size_t curr; size_t count; unsigned int ct; - msgpack_unpack_object map_key; + + union { + size_t curr; + msgpack_unpack_object map_key; + }; }; msgpack_unpack_struct_decl(_context) { From f61b282886bb9d7f7932fbcbe2affee06d150a5f Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:54:47 +0900 Subject: [PATCH 0182/1172] Reduce memory footprint. --- python/msgpack/unpack.h | 6 ++---- python/msgpack/unpack_template.h | 7 +++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index cc48d9c..12afb99 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -23,9 +23,7 @@ #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -using namespace std; - -typedef map str_cach_t; +typedef std::map str_cach_t; struct unpack_user { str_cach_t strcache; @@ -127,7 +125,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { if (l < 16) { - string s(p, l); + std::string s(p, l); str_cach_t ::iterator it = u->strcache.find(s); if (it != u->strcache.end()) { *o = it->second; diff --git a/python/msgpack/unpack_template.h b/python/msgpack/unpack_template.h index db33368..c960c3a 100644 --- a/python/msgpack/unpack_template.h +++ b/python/msgpack/unpack_template.h @@ -43,10 +43,13 @@ msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; - size_t curr; size_t count; unsigned int ct; - msgpack_unpack_object map_key; + + union { + size_t curr; + msgpack_unpack_object map_key; + }; }; msgpack_unpack_struct_decl(_context) { From 318b4b63ec295484e9fc8029a17b50f5181681ce Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 04:25:05 +0900 Subject: [PATCH 0183/1172] Check return value of c-api. --- msgpack/unpack.h | 124 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 95 insertions(+), 29 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 12afb99..e30b72a 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -59,42 +59,84 @@ static inline msgpack_unpack_object template_callback_root(unpack_user* u) return NULL; } -static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong((long)d); return 0; } - static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong((long)d); return 0; } +{ + PyObject *p = PyInt_FromLong((long)d); + if (!p) + return -1; + *o = p; + return 0; +} +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) +{ + return template_callback_uint16(u, d, o); +} + static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { + PyObject *p; if (d > LONG_MAX) { - *o = PyLong_FromUnsignedLong((unsigned long)d); + p = PyLong_FromUnsignedLong((unsigned long)d); } else { - *o = PyInt_FromLong((long)d); + p = PyInt_FromLong((long)d); } + if (!p) + return -1; + *o = p; return 0; } static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) -{ *o = PyLong_FromUnsignedLongLong(d); return 0; } - -static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } - -static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } +{ + PyObject *p = PyLong_FromUnsignedLongLong(d); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } +{ + PyObject *p = PyInt_FromLong(d); + if (!p) + return -1; + *o = p; + return 0; +} + +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) +{ + return template_callback_int32(u, d, o); +} + +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +{ + return template_callback_int32(u, d, o); +} static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) -{ *o = PyLong_FromLongLong(d); return 0; } - -static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) -{ *o = PyFloat_FromDouble((double)d); return 0; } +{ + PyObject *p = PyLong_FromLongLong(d); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) -{ *o = PyFloat_FromDouble(d); return 0; } +{ + PyObject *p = PyFloat_FromDouble(d); + if (!p) + return -1; + *o = p; + return 0; +} + +static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +{ + return template_callback_double(u, d, o); +} static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) { Py_INCREF(Py_None); *o = Py_None; return 0; } @@ -106,40 +148,64 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* { Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ *o = PyList_New(n); return 0; } +{ + PyObject *p = PyList_New(n); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) { PyList_SET_ITEM(*c, current, o); return 0; } static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ *o = PyDict_New(); return 0; } +{ + PyObject *p = PyDict_New(); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { - PyDict_SetItem(*c, k, v); - Py_DECREF(k); - Py_DECREF(v); - return 0; + if (PyDict_SetItem(*c, k, v) == 0) { + Py_DECREF(k); + Py_DECREF(v); + return 0; + } + return -1; } static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { + PyObject *py; if (l < 16) { std::string s(p, l); str_cach_t ::iterator it = u->strcache.find(s); if (it != u->strcache.end()) { *o = it->second; + Py_INCREF(*o); + return 0; } else { - *o = PyString_FromStringAndSize(p, l); + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + Py_INCREF(*o); u->strcache[s] = *o; + return 0; } - Py_INCREF(*o); } else { - *o = PyString_FromStringAndSize(p, l); + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + return 0; } - return 0; } #include "unpack_template.h" From 5d4189306ae0b6ea23e9410d34434b41aee384b3 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 04:25:05 +0900 Subject: [PATCH 0184/1172] Check return value of c-api. --- python/msgpack/unpack.h | 124 ++++++++++++++++++++++++++++++---------- 1 file changed, 95 insertions(+), 29 deletions(-) diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 12afb99..e30b72a 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -59,42 +59,84 @@ static inline msgpack_unpack_object template_callback_root(unpack_user* u) return NULL; } -static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong((long)d); return 0; } - static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong((long)d); return 0; } +{ + PyObject *p = PyInt_FromLong((long)d); + if (!p) + return -1; + *o = p; + return 0; +} +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) +{ + return template_callback_uint16(u, d, o); +} + static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { + PyObject *p; if (d > LONG_MAX) { - *o = PyLong_FromUnsignedLong((unsigned long)d); + p = PyLong_FromUnsignedLong((unsigned long)d); } else { - *o = PyInt_FromLong((long)d); + p = PyInt_FromLong((long)d); } + if (!p) + return -1; + *o = p; return 0; } static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) -{ *o = PyLong_FromUnsignedLongLong(d); return 0; } - -static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } - -static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } +{ + PyObject *p = PyLong_FromUnsignedLongLong(d); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } +{ + PyObject *p = PyInt_FromLong(d); + if (!p) + return -1; + *o = p; + return 0; +} + +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) +{ + return template_callback_int32(u, d, o); +} + +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +{ + return template_callback_int32(u, d, o); +} static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) -{ *o = PyLong_FromLongLong(d); return 0; } - -static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) -{ *o = PyFloat_FromDouble((double)d); return 0; } +{ + PyObject *p = PyLong_FromLongLong(d); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) -{ *o = PyFloat_FromDouble(d); return 0; } +{ + PyObject *p = PyFloat_FromDouble(d); + if (!p) + return -1; + *o = p; + return 0; +} + +static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +{ + return template_callback_double(u, d, o); +} static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) { Py_INCREF(Py_None); *o = Py_None; return 0; } @@ -106,40 +148,64 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* { Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ *o = PyList_New(n); return 0; } +{ + PyObject *p = PyList_New(n); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) { PyList_SET_ITEM(*c, current, o); return 0; } static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ *o = PyDict_New(); return 0; } +{ + PyObject *p = PyDict_New(); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { - PyDict_SetItem(*c, k, v); - Py_DECREF(k); - Py_DECREF(v); - return 0; + if (PyDict_SetItem(*c, k, v) == 0) { + Py_DECREF(k); + Py_DECREF(v); + return 0; + } + return -1; } static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { + PyObject *py; if (l < 16) { std::string s(p, l); str_cach_t ::iterator it = u->strcache.find(s); if (it != u->strcache.end()) { *o = it->second; + Py_INCREF(*o); + return 0; } else { - *o = PyString_FromStringAndSize(p, l); + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + Py_INCREF(*o); u->strcache[s] = *o; + return 0; } - Py_INCREF(*o); } else { - *o = PyString_FromStringAndSize(p, l); + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + return 0; } - return 0; } #include "unpack_template.h" From b6ba0d9baccf116ec29c5a0eb7605ae21d760740 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 14:33:36 +0900 Subject: [PATCH 0185/1172] Make setup.py executable. --- setup.py | 3 +++ 1 file changed, 3 insertions(+) mode change 100644 => 100755 setup.py diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 1cbc24e..e0e0ac2 --- a/setup.py +++ b/setup.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python +# coding: utf-8 + from distutils.core import setup, Extension from Cython.Distutils import build_ext import os From 1581acfd142b68ad4ddf56fbce6c7760f1a2e920 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 14:33:36 +0900 Subject: [PATCH 0186/1172] Make setup.py executable. --- python/setup.py | 3 +++ 1 file changed, 3 insertions(+) mode change 100644 => 100755 python/setup.py diff --git a/python/setup.py b/python/setup.py old mode 100644 new mode 100755 index 1cbc24e..e0e0ac2 --- a/python/setup.py +++ b/python/setup.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python +# coding: utf-8 + from distutils.core import setup, Extension from Cython.Distutils import build_ext import os From 8596d36d02b13424b18d3f95c9b80dd0006661ac Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 24 Jun 2009 13:53:12 +0900 Subject: [PATCH 0187/1172] perl package --- Makefile.am | 20 +++++++++++++++++++- c/msgpack.h | 1 + cpp/msgpack.hpp | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index d067fe0..9e36092 100644 --- a/Makefile.am +++ b/Makefile.am @@ -26,5 +26,23 @@ EXTRA_DIST = \ ruby/test_format.rb \ ruby/test_pack.rb \ ruby/unpack.c \ - ruby/unpack.h + ruby/unpack.h \ + perl/Makefile.PL \ + perl/MessagePack.c \ + perl/benchmark/deserialize.pl \ + perl/benchmark/serialize.pl \ + perl/lib/Data/MessagePack.pm \ + perl/lib/Data/MessagePack \ + perl/lib/Data/MessagePack/Unpacker.pod \ + perl/pack.c \ + perl/ppport.h \ + perl/t/00_compile.t \ + perl/t/01_pack.t \ + perl/t/02_unpack.t \ + perl/t/03_stream_unpack.t \ + perl/t/04_invert.t \ + perl/t/Util.pm \ + perl/t/data.pl \ + perl/unpack.c \ + perl/xt/99_pod.t diff --git a/c/msgpack.h b/c/msgpack.h index a59ef03..21729f4 100644 --- a/c/msgpack.h +++ b/c/msgpack.h @@ -20,3 +20,4 @@ #include "msgpack/pack.h" #include "msgpack/unpack.h" #include "msgpack/sbuffer.h" +#include "msgpack/vrefbuffer.h" diff --git a/cpp/msgpack.hpp b/cpp/msgpack.hpp index 58b40ac..e14680d 100644 --- a/cpp/msgpack.hpp +++ b/cpp/msgpack.hpp @@ -20,4 +20,5 @@ #include "msgpack/pack.hpp" #include "msgpack/unpack.hpp" #include "msgpack/sbuffer.hpp" +#include "msgpack/vrefbuffer.hpp" #include "msgpack.h" From a961f646e5dc1ea0c2881646ebf7373eb24ed564 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 24 Jun 2009 13:53:12 +0900 Subject: [PATCH 0188/1172] perl package --- Makefile.am | 20 +++++++++++++++++++- c/msgpack.h | 1 + cpp/msgpack.hpp | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index d067fe0..9e36092 100644 --- a/Makefile.am +++ b/Makefile.am @@ -26,5 +26,23 @@ EXTRA_DIST = \ ruby/test_format.rb \ ruby/test_pack.rb \ ruby/unpack.c \ - ruby/unpack.h + ruby/unpack.h \ + perl/Makefile.PL \ + perl/MessagePack.c \ + perl/benchmark/deserialize.pl \ + perl/benchmark/serialize.pl \ + perl/lib/Data/MessagePack.pm \ + perl/lib/Data/MessagePack \ + perl/lib/Data/MessagePack/Unpacker.pod \ + perl/pack.c \ + perl/ppport.h \ + perl/t/00_compile.t \ + perl/t/01_pack.t \ + perl/t/02_unpack.t \ + perl/t/03_stream_unpack.t \ + perl/t/04_invert.t \ + perl/t/Util.pm \ + perl/t/data.pl \ + perl/unpack.c \ + perl/xt/99_pod.t diff --git a/c/msgpack.h b/c/msgpack.h index a59ef03..21729f4 100644 --- a/c/msgpack.h +++ b/c/msgpack.h @@ -20,3 +20,4 @@ #include "msgpack/pack.h" #include "msgpack/unpack.h" #include "msgpack/sbuffer.h" +#include "msgpack/vrefbuffer.h" diff --git a/cpp/msgpack.hpp b/cpp/msgpack.hpp index 58b40ac..e14680d 100644 --- a/cpp/msgpack.hpp +++ b/cpp/msgpack.hpp @@ -20,4 +20,5 @@ #include "msgpack/pack.hpp" #include "msgpack/unpack.hpp" #include "msgpack/sbuffer.hpp" +#include "msgpack/vrefbuffer.hpp" #include "msgpack.h" From 9a342ab7f896cd3b6f66452278f80a424055b443 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 24 Jun 2009 14:11:06 +0900 Subject: [PATCH 0189/1172] update ppport.h --- perl/ppport.h | 95 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 8 deletions(-) diff --git a/perl/ppport.h b/perl/ppport.h index 3e3d52f..ec2f1cc 100644 --- a/perl/ppport.h +++ b/perl/ppport.h @@ -4,7 +4,7 @@ /* ---------------------------------------------------------------------- - ppport.h -- Perl/Pollution/Portability Version 3.17 + ppport.h -- Perl/Pollution/Portability Version 3.19 Automatically created by Devel::PPPort running under perl 5.010000. @@ -21,7 +21,7 @@ SKIP =head1 NAME -ppport.h - Perl/Pollution/Portability version 3.17 +ppport.h - Perl/Pollution/Portability version 3.19 =head1 SYNOPSIS @@ -232,6 +232,7 @@ same function or variable in your project. my_strlcpy() NEED_my_strlcpy NEED_my_strlcpy_GLOBAL newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL newRV_noinc() NEED_newRV_noinc NEED_newRV_noinc_GLOBAL + newSV_type() NEED_newSV_type NEED_newSV_type_GLOBAL newSVpvn_flags() NEED_newSVpvn_flags NEED_newSVpvn_flags_GLOBAL newSVpvn_share() NEED_newSVpvn_share NEED_newSVpvn_share_GLOBAL pv_display() NEED_pv_display NEED_pv_display_GLOBAL @@ -377,7 +378,7 @@ use strict; # Disable broken TRIE-optimization BEGIN { eval '${^RE_TRIE_MAXBUF} = -1' if $] >= 5.009004 && $] <= 5.009005 } -my $VERSION = 3.17; +my $VERSION = 3.19; my %opt = ( quiet => 0, @@ -486,6 +487,7 @@ G_NOARGS||| G_SCALAR||| G_VOID||5.004000| GetVars||| +GvSVn|5.009003||p GvSV||| Gv_AMupdate||| HEf_SVKEY||5.004000| @@ -498,6 +500,8 @@ HeSVKEY_set||5.004000| HeSVKEY||5.004000| HeUTF8||5.011000| HeVAL||5.004000| +HvNAMELEN_get|5.009003||p +HvNAME_get|5.009003||p HvNAME||| INT2PTR|5.006000||p IN_LOCALE_COMPILETIME|5.007002||p @@ -628,6 +632,9 @@ PERL_SHORT_MAX|5.004000||p PERL_SHORT_MIN|5.004000||p PERL_SIGNALS_UNSAFE_FLAG|5.008001||p PERL_SUBVERSION|5.006000||p +PERL_SYS_INIT3||5.006000| +PERL_SYS_INIT||| +PERL_SYS_TERM||5.011000| PERL_UCHAR_MAX|5.004000||p PERL_UCHAR_MIN|5.004000||p PERL_UINT_MAX|5.004000||p @@ -661,9 +668,12 @@ PL_diehook|5.004050||p PL_dirty|5.004050||p PL_dowarn|||pn PL_errgv|5.004050||p +PL_error_count|5.011000||p PL_expect|5.011000||p PL_hexdigit|5.005000||p PL_hints|5.005000||p +PL_in_my_stash|5.011000||p +PL_in_my|5.011000||p PL_last_in_gv|||n PL_laststatval|5.005000||p PL_lex_state|5.011000||p @@ -769,6 +779,7 @@ SV_MUTABLE_RETURN|5.009003||p SV_NOSTEAL|5.009002||p SV_SMAGIC|5.009003||p SV_UTF8_NO_ENCODING|5.008001||p +SVfARG|5.009005||p SVf_UTF8|5.006000||p SVf|5.006000||p SVt_IV||| @@ -977,6 +988,7 @@ XPUSHn||| XPUSHp||| XPUSHs||| XPUSHu|5.004000||p +XSPROTO|5.010000||p XSRETURN_EMPTY||| XSRETURN_IV||| XSRETURN_NO||| @@ -1055,7 +1067,6 @@ boolSV|5.004000||p boot_core_PerlIO||| boot_core_UNIVERSAL||| boot_core_mro||| -boot_core_xsutils||| bytes_from_utf8||5.007001| bytes_to_uni|||n bytes_to_utf8||5.006001| @@ -1341,7 +1352,6 @@ get_vtbl||5.005030| getcwd_sv||5.007002| getenv_len||| glob_2number||| -glob_2pv||| glob_assign_glob||| glob_assign_ref||| gp_dup||| @@ -1372,7 +1382,8 @@ gv_fetchmethod_autoload||5.004000| gv_fetchmethod_flags||5.011000| gv_fetchmethod||| gv_fetchmeth||| -gv_fetchpvn_flags||5.009002| +gv_fetchpvn_flags|5.009002||p +gv_fetchpvs|5.009004||p gv_fetchpv||| gv_fetchsv||5.009002| gv_fullname3||5.004000| @@ -1384,7 +1395,7 @@ gv_init_sv||| gv_init||| gv_name_set||5.009004| gv_stashpvn|5.004000||p -gv_stashpvs||5.009003| +gv_stashpvs|5.009003||p gv_stashpv||| gv_stashsv||| he_dup||| @@ -1470,6 +1481,7 @@ isBLANK|5.006001||p isCNTRL|5.006000||p isDIGIT||| isGRAPH|5.006000||p +isGV_with_GP|5.009004||p isLOWER||| isPRINT|5.004000||p isPSXSPC|5.006001||p @@ -1774,7 +1786,7 @@ newSTATEOP||| newSUB||| newSVOP||| newSVREF||| -newSV_type||5.009005| +newSV_type|5.009005||p newSVhek||5.009003| newSViv||| newSVnv||| @@ -2195,6 +2207,7 @@ sv_derived_from||5.004000| sv_destroyable||5.010000| sv_does||5.009004| sv_dump||| +sv_dup_inc_multiple||| sv_dup||| sv_eq||| sv_exp_grow||| @@ -3907,6 +3920,13 @@ typedef NVTYPE NV; return; \ } STMT_END #endif +#ifndef XSPROTO +# define XSPROTO(name) void name(pTHX_ CV* cv) +#endif + +#ifndef SVfARG +# define SVfARG(p) ((void*)(p)) +#endif #ifndef PERL_ABS # define PERL_ABS(x) ((x) < 0 ? -(x) : (x)) #endif @@ -4086,9 +4106,11 @@ extern U32 DPPP_(my_PL_signals); # define PL_dirty dirty # define PL_dowarn dowarn # define PL_errgv errgv +# define PL_error_count error_count # define PL_expect expect # define PL_hexdigit hexdigit # define PL_hints hints +# define PL_in_my in_my # define PL_laststatval laststatval # define PL_lex_state lex_state # define PL_lex_stuff lex_stuff @@ -4171,6 +4193,10 @@ extern yy_parser DPPP_(dummy_PL_parser); # define PL_lex_state D_PPP_my_PL_parser_var(lex_state) # define PL_lex_stuff D_PPP_my_PL_parser_var(lex_stuff) # define PL_tokenbuf D_PPP_my_PL_parser_var(tokenbuf) +# define PL_in_my D_PPP_my_PL_parser_var(in_my) +# define PL_in_my_stash D_PPP_my_PL_parser_var(in_my_stash) +# define PL_error_count D_PPP_my_PL_parser_var(error_count) + #else @@ -4711,6 +4737,35 @@ DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv) # define SvREFCNT_inc_simple_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) #endif +#ifndef newSV_type + +#if defined(NEED_newSV_type) +static SV* DPPP_(my_newSV_type)(pTHX_ svtype const t); +static +#else +extern SV* DPPP_(my_newSV_type)(pTHX_ svtype const t); +#endif + +#ifdef newSV_type +# undef newSV_type +#endif +#define newSV_type(a) DPPP_(my_newSV_type)(aTHX_ a) +#define Perl_newSV_type DPPP_(my_newSV_type) + +#if defined(NEED_newSV_type) || defined(NEED_newSV_type_GLOBAL) + +SV* +DPPP_(my_newSV_type)(pTHX_ svtype const t) +{ + SV* const sv = newSV(0); + sv_upgrade(sv, t); + return sv; +} + +#endif + +#endif + #if (PERL_BCDVERSION < 0x5006000) # define D_PPP_CONSTPV_ARG(x) ((char *) (x)) #else @@ -5298,6 +5353,19 @@ DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash) #ifndef SvSHARED_HASH # define SvSHARED_HASH(sv) (0 + SvUVX(sv)) #endif +#ifndef HvNAME_get +# define HvNAME_get(hv) HvNAME(hv) +#endif +#ifndef HvNAMELEN_get +# define HvNAMELEN_get(hv) (HvNAME_get(hv) ? (I32)strlen(HvNAME_get(hv)) : 0) +#endif +#ifndef GvSVn +# define GvSVn(gv) GvSV(gv) +#endif + +#ifndef isGV_with_GP +# define isGV_with_GP(gv) isGV(gv) +#endif #ifndef WARN_ALL # define WARN_ALL 0 #endif @@ -5561,6 +5629,17 @@ DPPP_(my_warner)(U32 err, const char *pat, ...) #ifndef hv_stores # define hv_stores(hv, key, val) hv_store(hv, key "", sizeof(key) - 1, val, 0) #endif +#ifndef gv_fetchpvn_flags +# define gv_fetchpvn_flags(name, len, flags, svt) gv_fetchpv(name, flags, svt) +#endif + +#ifndef gv_fetchpvs +# define gv_fetchpvs(name, flags, svt) gv_fetchpvn_flags(name "", sizeof(name) - 1, flags, svt) +#endif + +#ifndef gv_stashpvs +# define gv_stashpvs(name, flags) gv_stashpvn(name "", sizeof(name) - 1, flags) +#endif #ifndef SvGETMAGIC # define SvGETMAGIC(x) STMT_START { if (SvGMAGICAL(x)) mg_get(x); } STMT_END #endif From b2fdaac99efc9a6c43bcc87f59103ad98d2c5bd3 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 24 Jun 2009 14:11:06 +0900 Subject: [PATCH 0190/1172] update ppport.h --- perl/ppport.h | 95 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 8 deletions(-) diff --git a/perl/ppport.h b/perl/ppport.h index 3e3d52f..ec2f1cc 100644 --- a/perl/ppport.h +++ b/perl/ppport.h @@ -4,7 +4,7 @@ /* ---------------------------------------------------------------------- - ppport.h -- Perl/Pollution/Portability Version 3.17 + ppport.h -- Perl/Pollution/Portability Version 3.19 Automatically created by Devel::PPPort running under perl 5.010000. @@ -21,7 +21,7 @@ SKIP =head1 NAME -ppport.h - Perl/Pollution/Portability version 3.17 +ppport.h - Perl/Pollution/Portability version 3.19 =head1 SYNOPSIS @@ -232,6 +232,7 @@ same function or variable in your project. my_strlcpy() NEED_my_strlcpy NEED_my_strlcpy_GLOBAL newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL newRV_noinc() NEED_newRV_noinc NEED_newRV_noinc_GLOBAL + newSV_type() NEED_newSV_type NEED_newSV_type_GLOBAL newSVpvn_flags() NEED_newSVpvn_flags NEED_newSVpvn_flags_GLOBAL newSVpvn_share() NEED_newSVpvn_share NEED_newSVpvn_share_GLOBAL pv_display() NEED_pv_display NEED_pv_display_GLOBAL @@ -377,7 +378,7 @@ use strict; # Disable broken TRIE-optimization BEGIN { eval '${^RE_TRIE_MAXBUF} = -1' if $] >= 5.009004 && $] <= 5.009005 } -my $VERSION = 3.17; +my $VERSION = 3.19; my %opt = ( quiet => 0, @@ -486,6 +487,7 @@ G_NOARGS||| G_SCALAR||| G_VOID||5.004000| GetVars||| +GvSVn|5.009003||p GvSV||| Gv_AMupdate||| HEf_SVKEY||5.004000| @@ -498,6 +500,8 @@ HeSVKEY_set||5.004000| HeSVKEY||5.004000| HeUTF8||5.011000| HeVAL||5.004000| +HvNAMELEN_get|5.009003||p +HvNAME_get|5.009003||p HvNAME||| INT2PTR|5.006000||p IN_LOCALE_COMPILETIME|5.007002||p @@ -628,6 +632,9 @@ PERL_SHORT_MAX|5.004000||p PERL_SHORT_MIN|5.004000||p PERL_SIGNALS_UNSAFE_FLAG|5.008001||p PERL_SUBVERSION|5.006000||p +PERL_SYS_INIT3||5.006000| +PERL_SYS_INIT||| +PERL_SYS_TERM||5.011000| PERL_UCHAR_MAX|5.004000||p PERL_UCHAR_MIN|5.004000||p PERL_UINT_MAX|5.004000||p @@ -661,9 +668,12 @@ PL_diehook|5.004050||p PL_dirty|5.004050||p PL_dowarn|||pn PL_errgv|5.004050||p +PL_error_count|5.011000||p PL_expect|5.011000||p PL_hexdigit|5.005000||p PL_hints|5.005000||p +PL_in_my_stash|5.011000||p +PL_in_my|5.011000||p PL_last_in_gv|||n PL_laststatval|5.005000||p PL_lex_state|5.011000||p @@ -769,6 +779,7 @@ SV_MUTABLE_RETURN|5.009003||p SV_NOSTEAL|5.009002||p SV_SMAGIC|5.009003||p SV_UTF8_NO_ENCODING|5.008001||p +SVfARG|5.009005||p SVf_UTF8|5.006000||p SVf|5.006000||p SVt_IV||| @@ -977,6 +988,7 @@ XPUSHn||| XPUSHp||| XPUSHs||| XPUSHu|5.004000||p +XSPROTO|5.010000||p XSRETURN_EMPTY||| XSRETURN_IV||| XSRETURN_NO||| @@ -1055,7 +1067,6 @@ boolSV|5.004000||p boot_core_PerlIO||| boot_core_UNIVERSAL||| boot_core_mro||| -boot_core_xsutils||| bytes_from_utf8||5.007001| bytes_to_uni|||n bytes_to_utf8||5.006001| @@ -1341,7 +1352,6 @@ get_vtbl||5.005030| getcwd_sv||5.007002| getenv_len||| glob_2number||| -glob_2pv||| glob_assign_glob||| glob_assign_ref||| gp_dup||| @@ -1372,7 +1382,8 @@ gv_fetchmethod_autoload||5.004000| gv_fetchmethod_flags||5.011000| gv_fetchmethod||| gv_fetchmeth||| -gv_fetchpvn_flags||5.009002| +gv_fetchpvn_flags|5.009002||p +gv_fetchpvs|5.009004||p gv_fetchpv||| gv_fetchsv||5.009002| gv_fullname3||5.004000| @@ -1384,7 +1395,7 @@ gv_init_sv||| gv_init||| gv_name_set||5.009004| gv_stashpvn|5.004000||p -gv_stashpvs||5.009003| +gv_stashpvs|5.009003||p gv_stashpv||| gv_stashsv||| he_dup||| @@ -1470,6 +1481,7 @@ isBLANK|5.006001||p isCNTRL|5.006000||p isDIGIT||| isGRAPH|5.006000||p +isGV_with_GP|5.009004||p isLOWER||| isPRINT|5.004000||p isPSXSPC|5.006001||p @@ -1774,7 +1786,7 @@ newSTATEOP||| newSUB||| newSVOP||| newSVREF||| -newSV_type||5.009005| +newSV_type|5.009005||p newSVhek||5.009003| newSViv||| newSVnv||| @@ -2195,6 +2207,7 @@ sv_derived_from||5.004000| sv_destroyable||5.010000| sv_does||5.009004| sv_dump||| +sv_dup_inc_multiple||| sv_dup||| sv_eq||| sv_exp_grow||| @@ -3907,6 +3920,13 @@ typedef NVTYPE NV; return; \ } STMT_END #endif +#ifndef XSPROTO +# define XSPROTO(name) void name(pTHX_ CV* cv) +#endif + +#ifndef SVfARG +# define SVfARG(p) ((void*)(p)) +#endif #ifndef PERL_ABS # define PERL_ABS(x) ((x) < 0 ? -(x) : (x)) #endif @@ -4086,9 +4106,11 @@ extern U32 DPPP_(my_PL_signals); # define PL_dirty dirty # define PL_dowarn dowarn # define PL_errgv errgv +# define PL_error_count error_count # define PL_expect expect # define PL_hexdigit hexdigit # define PL_hints hints +# define PL_in_my in_my # define PL_laststatval laststatval # define PL_lex_state lex_state # define PL_lex_stuff lex_stuff @@ -4171,6 +4193,10 @@ extern yy_parser DPPP_(dummy_PL_parser); # define PL_lex_state D_PPP_my_PL_parser_var(lex_state) # define PL_lex_stuff D_PPP_my_PL_parser_var(lex_stuff) # define PL_tokenbuf D_PPP_my_PL_parser_var(tokenbuf) +# define PL_in_my D_PPP_my_PL_parser_var(in_my) +# define PL_in_my_stash D_PPP_my_PL_parser_var(in_my_stash) +# define PL_error_count D_PPP_my_PL_parser_var(error_count) + #else @@ -4711,6 +4737,35 @@ DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv) # define SvREFCNT_inc_simple_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) #endif +#ifndef newSV_type + +#if defined(NEED_newSV_type) +static SV* DPPP_(my_newSV_type)(pTHX_ svtype const t); +static +#else +extern SV* DPPP_(my_newSV_type)(pTHX_ svtype const t); +#endif + +#ifdef newSV_type +# undef newSV_type +#endif +#define newSV_type(a) DPPP_(my_newSV_type)(aTHX_ a) +#define Perl_newSV_type DPPP_(my_newSV_type) + +#if defined(NEED_newSV_type) || defined(NEED_newSV_type_GLOBAL) + +SV* +DPPP_(my_newSV_type)(pTHX_ svtype const t) +{ + SV* const sv = newSV(0); + sv_upgrade(sv, t); + return sv; +} + +#endif + +#endif + #if (PERL_BCDVERSION < 0x5006000) # define D_PPP_CONSTPV_ARG(x) ((char *) (x)) #else @@ -5298,6 +5353,19 @@ DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash) #ifndef SvSHARED_HASH # define SvSHARED_HASH(sv) (0 + SvUVX(sv)) #endif +#ifndef HvNAME_get +# define HvNAME_get(hv) HvNAME(hv) +#endif +#ifndef HvNAMELEN_get +# define HvNAMELEN_get(hv) (HvNAME_get(hv) ? (I32)strlen(HvNAME_get(hv)) : 0) +#endif +#ifndef GvSVn +# define GvSVn(gv) GvSV(gv) +#endif + +#ifndef isGV_with_GP +# define isGV_with_GP(gv) isGV(gv) +#endif #ifndef WARN_ALL # define WARN_ALL 0 #endif @@ -5561,6 +5629,17 @@ DPPP_(my_warner)(U32 err, const char *pat, ...) #ifndef hv_stores # define hv_stores(hv, key, val) hv_store(hv, key "", sizeof(key) - 1, val, 0) #endif +#ifndef gv_fetchpvn_flags +# define gv_fetchpvn_flags(name, len, flags, svt) gv_fetchpv(name, flags, svt) +#endif + +#ifndef gv_fetchpvs +# define gv_fetchpvs(name, flags, svt) gv_fetchpvn_flags(name "", sizeof(name) - 1, flags, svt) +#endif + +#ifndef gv_stashpvs +# define gv_stashpvs(name, flags) gv_stashpvn(name "", sizeof(name) - 1, flags) +#endif #ifndef SvGETMAGIC # define SvGETMAGIC(x) STMT_START { if (SvGMAGICAL(x)) mg_get(x); } STMT_END #endif From 3aef3edbad682ecd38ae0bd83cfb1060800335a4 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 24 Jun 2009 14:36:37 +0900 Subject: [PATCH 0191/1172] works correctly under Perl5.8.x. Perl5.8.x makes "a" as PVIV in {"a", 0}. --- perl/pack.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index 9c32dc1..5aec963 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -81,14 +81,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { } } break; - case SVt_PV: - { - STRLEN len; - char * cval = SvPV(val, len); - msgpack_pack_raw(enc, len); - msgpack_pack_raw_body(enc, cval, len); - } - break; case SVt_NV: PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); break; @@ -126,8 +118,16 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { _msgpack_pack_sv(enc, SvRV(val)); break; default: - sv_dump(val); - Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); + if (SvPOKp(val)) { + STRLEN len; + char * cval = SvPV(val, len); + msgpack_pack_raw(enc, len); + msgpack_pack_raw_body(enc, cval, len); + return; + } else { + sv_dump(val); + Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); + } } } From 8e166de1d1b6279dfe94583631711fde5bb22bea Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 24 Jun 2009 14:46:28 +0900 Subject: [PATCH 0192/1172] works correctly under Perl5.8.x. Perl5.8.x makes "a" as PVIV in {"a", 0}. --- perl/pack.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index 9c32dc1..5aec963 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -81,14 +81,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { } } break; - case SVt_PV: - { - STRLEN len; - char * cval = SvPV(val, len); - msgpack_pack_raw(enc, len); - msgpack_pack_raw_body(enc, cval, len); - } - break; case SVt_NV: PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); break; @@ -126,8 +118,16 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { _msgpack_pack_sv(enc, SvRV(val)); break; default: - sv_dump(val); - Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); + if (SvPOKp(val)) { + STRLEN len; + char * cval = SvPV(val, len); + msgpack_pack_raw(enc, len); + msgpack_pack_raw_body(enc, cval, len); + return; + } else { + sv_dump(val); + Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); + } } } From 87e1adcb40ae4946c7443a7c1483a3e521869052 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 24 Jun 2009 14:36:37 +0900 Subject: [PATCH 0193/1172] works correctly under Perl5.8.x. Perl5.8.x makes "a" as PVIV in {"a", 0}. --- perl/pack.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index 9c32dc1..5aec963 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -81,14 +81,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { } } break; - case SVt_PV: - { - STRLEN len; - char * cval = SvPV(val, len); - msgpack_pack_raw(enc, len); - msgpack_pack_raw_body(enc, cval, len); - } - break; case SVt_NV: PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); break; @@ -126,8 +118,16 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { _msgpack_pack_sv(enc, SvRV(val)); break; default: - sv_dump(val); - Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); + if (SvPOKp(val)) { + STRLEN len; + char * cval = SvPV(val, len); + msgpack_pack_raw(enc, len); + msgpack_pack_raw_body(enc, cval, len); + return; + } else { + sv_dump(val); + Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); + } } } From 479263989b02693f0e449ee591ca58ed84fd904b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 14:58:02 +0900 Subject: [PATCH 0194/1172] Stop unnecessary caching. --- python/msgpack/unpack.h | 44 +++++------------------------------------ 1 file changed, 5 insertions(+), 39 deletions(-) diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index e30b72a..dd23130 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -16,24 +16,10 @@ * limitations under the License. */ -#include -#include -#include - #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -typedef std::map str_cach_t; struct unpack_user { - str_cach_t strcache; - - ~unpack_user() { - str_cach_t::iterator it, itend; - itend = strcache.end(); - for (it = strcache.begin(); it != itend; ++it) { - Py_DECREF(it->second); - } - } }; @@ -181,31 +167,11 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { PyObject *py; - if (l < 16) { - std::string s(p, l); - str_cach_t ::iterator it = u->strcache.find(s); - if (it != u->strcache.end()) { - *o = it->second; - Py_INCREF(*o); - return 0; - } - else { - py = PyString_FromStringAndSize(p, l); - if (!py) - return -1; - *o = py; - Py_INCREF(*o); - u->strcache[s] = *o; - return 0; - } - } - else { - py = PyString_FromStringAndSize(p, l); - if (!py) - return -1; - *o = py; - return 0; - } + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + return 0; } #include "unpack_template.h" From b944eefb96a0f3ad907c8fe704fd14ba87384ed5 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 14:58:02 +0900 Subject: [PATCH 0195/1172] Stop unnecessary caching. --- msgpack/unpack.h | 44 +++++--------------------------------------- 1 file changed, 5 insertions(+), 39 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index e30b72a..dd23130 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -16,24 +16,10 @@ * limitations under the License. */ -#include -#include -#include - #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -typedef std::map str_cach_t; struct unpack_user { - str_cach_t strcache; - - ~unpack_user() { - str_cach_t::iterator it, itend; - itend = strcache.end(); - for (it = strcache.begin(); it != itend; ++it) { - Py_DECREF(it->second); - } - } }; @@ -181,31 +167,11 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { PyObject *py; - if (l < 16) { - std::string s(p, l); - str_cach_t ::iterator it = u->strcache.find(s); - if (it != u->strcache.end()) { - *o = it->second; - Py_INCREF(*o); - return 0; - } - else { - py = PyString_FromStringAndSize(p, l); - if (!py) - return -1; - *o = py; - Py_INCREF(*o); - u->strcache[s] = *o; - return 0; - } - } - else { - py = PyString_FromStringAndSize(p, l); - if (!py) - return -1; - *o = py; - return 0; - } + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + return 0; } #include "unpack_template.h" From 8f52ed26c7ecbc28ebcc8deb3f65bdce940fd22b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 24 Jun 2009 16:01:13 +0900 Subject: [PATCH 0196/1172] version 0.3.3 --- configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 47b28cf..45a04a9 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.2) +AM_INIT_AUTOMAKE(msgpack, 0.3.3) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) From dd407410d25b4364897f9518f0fd173304b4718b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 24 Jun 2009 16:01:13 +0900 Subject: [PATCH 0197/1172] version 0.3.3 --- configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 47b28cf..45a04a9 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.2) +AM_INIT_AUTOMAKE(msgpack, 0.3.3) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) From 3c3df3133c6f6320a03ebcaef546ab9c2c53154d Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Fri, 26 Jun 2009 14:10:20 +0900 Subject: [PATCH 0198/1172] Implement streaming deserializer. --- python/msgpack/_msgpack.pyx | 162 ++++++++++++++++++++++++++++++++---- python/test_sequnpack.py | 36 ++++++++ 2 files changed, 183 insertions(+), 15 deletions(-) create mode 100644 python/test_sequnpack.py diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index cbdcfc5..c37f8ba 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -6,13 +6,16 @@ cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + char* PyString_AsString(object o) cdef extern from "stdlib.h": - void* malloc(int) + void* malloc(size_t) + void* realloc(void*, size_t) void free(void*) cdef extern from "string.h": - int memcpy(char*dst, char*src, unsigned int size) + void* memcpy(char* dst, char* src, size_t size) + void* memmove(char* dst, char* src, size_t size) cdef extern from "pack.h": ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) @@ -34,8 +37,6 @@ cdef extern from "pack.h": void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef int BUFF_SIZE=2*1024 - cdef class Packer: """Packer that pack data into strm. @@ -48,10 +49,7 @@ cdef class Packer: cdef msgpack_packer pk cdef object strm - def __init__(self, strm, int size=0): - if size <= 0: - size = BUFF_SIZE - + def __init__(self, strm, int size=4*1024): self.strm = strm self.buff = malloc(size) self.allocated = size @@ -147,6 +145,8 @@ cdef class Packer: if flush: self.flush() + close = flush + cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): if packer.length + l > packer.allocated: if packer.length > 0: @@ -163,20 +163,28 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): return 0 def pack(object o, object stream): + u"""pack o and write to stream).""" packer = Packer(stream) packer.pack(o) packer.flush() -def packs(object o): +def packb(object o): + u"""pack o and return packed bytes.""" buf = StringIO() packer = Packer(buf) packer.pack(o) packer.flush() return buf.getvalue() +packs = packb + cdef extern from "unpack.h": ctypedef struct template_context: - pass + PyObject* obj + size_t count + unsigned int ct + PyObject* key + int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) void template_init(template_context* ctx) @@ -188,15 +196,139 @@ def unpacks(object packed_bytes): cdef const_char_ptr p = packed_bytes cdef template_context ctx cdef size_t off = 0 + cdef int ret template_init(&ctx) - template_execute(&ctx, p, len(packed_bytes), &off) - return template_data(&ctx) + ret = template_execute(&ctx, p, len(packed_bytes), &off) + if ret == 1: + return template_data(&ctx) + else: + return None def unpack(object stream): """unpack from stream.""" packed = stream.read() return unpacks(packed) -cdef class Unpacker: - """Do nothing. This function is for symmetric to Packer""" - unpack = staticmethod(unpacks) +cdef class UnpackIterator(object): + cdef object unpacker + + def __init__(self, unpacker): + self.unpacker = unpacker + + def __next__(self): + return self.unpacker.unpack() + + def __iter__(self): + return self + +cdef class Unpacker(object): + """Unpacker(file_like=None, read_size=4096) + + Streaming unpacker. + file_like must have read(n) method. + read_size is used like file_like.read(read_size) + + If file_like is None, you can feed() bytes. feed() is useful + for unpack from non-blocking stream. + + exsample 1: + unpacker = Unpacker(afile) + for o in unpacker: + do_something(o) + + example 2: + unpacker = Unpacker() + while 1: + buf = astream.read() + unpacker.feed(buf) + for o in unpacker: + do_something(o) + """ + + cdef template_context ctx + cdef char* buf + cdef size_t buf_size, buf_head, buf_tail + cdef object file_like + cdef int read_size + cdef object waiting_bytes + + def __init__(self, file_like=None, int read_size=4096): + self.file_like = file_like + self.read_size = read_size + self.waiting_bytes = [] + self.buf = malloc(read_size) + self.buf_size = read_size + self.buf_head = 0 + self.buf_tail = 0 + template_init(&self.ctx) + + def feed(self, next_bytes): + if not isinstance(next_bytes, str): + raise ValueError, "Argument must be bytes object" + self.waiting_bytes.append(next_bytes) + + cdef append_buffer(self): + cdef char* buf = self.buf + cdef Py_ssize_t tail = self.buf_tail + cdef Py_ssize_t l + + for b in self.waiting_bytes: + l = len(b) + memcpy(buf + tail, PyString_AsString(b), l) + tail += l + self.buf_tail = tail + del self.waiting_bytes[:] + + # prepare self.buf + cdef fill_buffer(self): + cdef Py_ssize_t add_size + + if self.file_like is not None: + self.waiting_bytes.append(self.file_like.read(self.read_size)) + + if not self.waiting_bytes: + return + + add_size = 0 + for b in self.waiting_bytes: + add_size += len(b) + + cdef char* buf = self.buf + cdef size_t head = self.buf_head + cdef size_t tail = self.buf_tail + cdef size_t size = self.buf_size + + if self.buf_tail + add_size <= self.buf_size: + # do nothing. + pass + if self.buf_tail - self.buf_head + add_size < self.buf_size: + # move to front. + memmove(buf, buf + head, tail - head) + tail -= head + head = 0 + else: + # expand buffer + size = tail + add_size + buf = realloc(buf, size) + + self.buf = buf + self.buf_head = head + self.buf_tail = tail + self.buf_size = size + + self.append_buffer() + + cpdef unpack(self): + """unpack one object""" + cdef int ret + self.fill_buffer() + ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + if ret == 1: + return template_data(&self.ctx) + elif ret == 0: + raise StopIteration, "No more unpack data." + else: + raise ValueError, "Unpack failed." + + def __iter__(self): + return UnpackIterator(self) diff --git a/python/test_sequnpack.py b/python/test_sequnpack.py new file mode 100644 index 0000000..789ccd2 --- /dev/null +++ b/python/test_sequnpack.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# coding: utf-8 + +from __future__ import unicode_literals, print_function + +from msgpack import Unpacker + +def test_foobar(): + unpacker = Unpacker(read_size=3) + unpacker.feed(b'foobar') + assert unpacker.unpack() == ord('f') + assert unpacker.unpack() == ord('o') + assert unpacker.unpack() == ord('o') + assert unpacker.unpack() == ord('b') + assert unpacker.unpack() == ord('a') + assert unpacker.unpack() == ord('r') + try: + o = unpacker.unpack() + print("Oops!", o) + assert 0 + except StopIteration: + assert 1 + else: + assert 0 + unpacker.feed(b'foo') + unpacker.feed(b'bar') + + k = 0 + for o, e in zip(unpacker, b'foobarbaz'): + assert o == ord(e) + k += 1 + assert k == len(b'foobar') + +if __name__ == '__main__': + test_foobar() + From 1781092bd8da6e014f78873516dd1e403205fcd7 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Fri, 26 Jun 2009 14:10:20 +0900 Subject: [PATCH 0199/1172] Implement streaming deserializer. --- msgpack/_msgpack.pyx | 162 +++++++++++++++++++++++++++++++++++++++---- test_sequnpack.py | 36 ++++++++++ 2 files changed, 183 insertions(+), 15 deletions(-) create mode 100644 test_sequnpack.py diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index cbdcfc5..c37f8ba 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -6,13 +6,16 @@ cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + char* PyString_AsString(object o) cdef extern from "stdlib.h": - void* malloc(int) + void* malloc(size_t) + void* realloc(void*, size_t) void free(void*) cdef extern from "string.h": - int memcpy(char*dst, char*src, unsigned int size) + void* memcpy(char* dst, char* src, size_t size) + void* memmove(char* dst, char* src, size_t size) cdef extern from "pack.h": ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) @@ -34,8 +37,6 @@ cdef extern from "pack.h": void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef int BUFF_SIZE=2*1024 - cdef class Packer: """Packer that pack data into strm. @@ -48,10 +49,7 @@ cdef class Packer: cdef msgpack_packer pk cdef object strm - def __init__(self, strm, int size=0): - if size <= 0: - size = BUFF_SIZE - + def __init__(self, strm, int size=4*1024): self.strm = strm self.buff = malloc(size) self.allocated = size @@ -147,6 +145,8 @@ cdef class Packer: if flush: self.flush() + close = flush + cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): if packer.length + l > packer.allocated: if packer.length > 0: @@ -163,20 +163,28 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): return 0 def pack(object o, object stream): + u"""pack o and write to stream).""" packer = Packer(stream) packer.pack(o) packer.flush() -def packs(object o): +def packb(object o): + u"""pack o and return packed bytes.""" buf = StringIO() packer = Packer(buf) packer.pack(o) packer.flush() return buf.getvalue() +packs = packb + cdef extern from "unpack.h": ctypedef struct template_context: - pass + PyObject* obj + size_t count + unsigned int ct + PyObject* key + int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) void template_init(template_context* ctx) @@ -188,15 +196,139 @@ def unpacks(object packed_bytes): cdef const_char_ptr p = packed_bytes cdef template_context ctx cdef size_t off = 0 + cdef int ret template_init(&ctx) - template_execute(&ctx, p, len(packed_bytes), &off) - return template_data(&ctx) + ret = template_execute(&ctx, p, len(packed_bytes), &off) + if ret == 1: + return template_data(&ctx) + else: + return None def unpack(object stream): """unpack from stream.""" packed = stream.read() return unpacks(packed) -cdef class Unpacker: - """Do nothing. This function is for symmetric to Packer""" - unpack = staticmethod(unpacks) +cdef class UnpackIterator(object): + cdef object unpacker + + def __init__(self, unpacker): + self.unpacker = unpacker + + def __next__(self): + return self.unpacker.unpack() + + def __iter__(self): + return self + +cdef class Unpacker(object): + """Unpacker(file_like=None, read_size=4096) + + Streaming unpacker. + file_like must have read(n) method. + read_size is used like file_like.read(read_size) + + If file_like is None, you can feed() bytes. feed() is useful + for unpack from non-blocking stream. + + exsample 1: + unpacker = Unpacker(afile) + for o in unpacker: + do_something(o) + + example 2: + unpacker = Unpacker() + while 1: + buf = astream.read() + unpacker.feed(buf) + for o in unpacker: + do_something(o) + """ + + cdef template_context ctx + cdef char* buf + cdef size_t buf_size, buf_head, buf_tail + cdef object file_like + cdef int read_size + cdef object waiting_bytes + + def __init__(self, file_like=None, int read_size=4096): + self.file_like = file_like + self.read_size = read_size + self.waiting_bytes = [] + self.buf = malloc(read_size) + self.buf_size = read_size + self.buf_head = 0 + self.buf_tail = 0 + template_init(&self.ctx) + + def feed(self, next_bytes): + if not isinstance(next_bytes, str): + raise ValueError, "Argument must be bytes object" + self.waiting_bytes.append(next_bytes) + + cdef append_buffer(self): + cdef char* buf = self.buf + cdef Py_ssize_t tail = self.buf_tail + cdef Py_ssize_t l + + for b in self.waiting_bytes: + l = len(b) + memcpy(buf + tail, PyString_AsString(b), l) + tail += l + self.buf_tail = tail + del self.waiting_bytes[:] + + # prepare self.buf + cdef fill_buffer(self): + cdef Py_ssize_t add_size + + if self.file_like is not None: + self.waiting_bytes.append(self.file_like.read(self.read_size)) + + if not self.waiting_bytes: + return + + add_size = 0 + for b in self.waiting_bytes: + add_size += len(b) + + cdef char* buf = self.buf + cdef size_t head = self.buf_head + cdef size_t tail = self.buf_tail + cdef size_t size = self.buf_size + + if self.buf_tail + add_size <= self.buf_size: + # do nothing. + pass + if self.buf_tail - self.buf_head + add_size < self.buf_size: + # move to front. + memmove(buf, buf + head, tail - head) + tail -= head + head = 0 + else: + # expand buffer + size = tail + add_size + buf = realloc(buf, size) + + self.buf = buf + self.buf_head = head + self.buf_tail = tail + self.buf_size = size + + self.append_buffer() + + cpdef unpack(self): + """unpack one object""" + cdef int ret + self.fill_buffer() + ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + if ret == 1: + return template_data(&self.ctx) + elif ret == 0: + raise StopIteration, "No more unpack data." + else: + raise ValueError, "Unpack failed." + + def __iter__(self): + return UnpackIterator(self) diff --git a/test_sequnpack.py b/test_sequnpack.py new file mode 100644 index 0000000..789ccd2 --- /dev/null +++ b/test_sequnpack.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# coding: utf-8 + +from __future__ import unicode_literals, print_function + +from msgpack import Unpacker + +def test_foobar(): + unpacker = Unpacker(read_size=3) + unpacker.feed(b'foobar') + assert unpacker.unpack() == ord('f') + assert unpacker.unpack() == ord('o') + assert unpacker.unpack() == ord('o') + assert unpacker.unpack() == ord('b') + assert unpacker.unpack() == ord('a') + assert unpacker.unpack() == ord('r') + try: + o = unpacker.unpack() + print("Oops!", o) + assert 0 + except StopIteration: + assert 1 + else: + assert 0 + unpacker.feed(b'foo') + unpacker.feed(b'bar') + + k = 0 + for o, e in zip(unpacker, b'foobarbaz'): + assert o == ord(e) + k += 1 + assert k == len(b'foobar') + +if __name__ == '__main__': + test_foobar() + From a345131aaa4a87c2e917913eae053a8382fd4e67 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sat, 27 Jun 2009 12:03:00 +0900 Subject: [PATCH 0200/1172] Add: README --- python/README | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 python/README diff --git a/python/README b/python/README new file mode 100644 index 0000000..12a27b2 --- /dev/null +++ b/python/README @@ -0,0 +1,31 @@ +=========================== +MessagePack Python Binding +=========================== + +:author: Naoki INADA +:version: 0.0.1 +:date: 2009-06-27 + +HOW TO USE +----------- +You can read document in docstring after `import msgpack` + + +INSTALL +--------- +Cython_ is required to build the binding. + +.. _Cython: http://www.cython.org/ + +Posix +'''''' +You can install msgpack in common way. + + $ python setup.py install + +Windows +'''''''' +MessagePack requires gcc currently. So you need to prepare +MinGW GCC. + + $ python setup.py install -c mingw From 26a8d5d90f487728f2ab1d3cd0289500d68300e1 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sat, 27 Jun 2009 12:03:00 +0900 Subject: [PATCH 0201/1172] Add: README --- README | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 README diff --git a/README b/README new file mode 100644 index 0000000..12a27b2 --- /dev/null +++ b/README @@ -0,0 +1,31 @@ +=========================== +MessagePack Python Binding +=========================== + +:author: Naoki INADA +:version: 0.0.1 +:date: 2009-06-27 + +HOW TO USE +----------- +You can read document in docstring after `import msgpack` + + +INSTALL +--------- +Cython_ is required to build the binding. + +.. _Cython: http://www.cython.org/ + +Posix +'''''' +You can install msgpack in common way. + + $ python setup.py install + +Windows +'''''''' +MessagePack requires gcc currently. So you need to prepare +MinGW GCC. + + $ python setup.py install -c mingw From fa2efcdb5ba8a5cca141f7882271abb8ea697caa Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sat, 27 Jun 2009 12:04:11 +0900 Subject: [PATCH 0202/1172] Rename test files. --- python/{testformat.py => test/test_format.py} | 0 python/{ => test}/test_sequnpack.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename python/{testformat.py => test/test_format.py} (100%) rename python/{ => test}/test_sequnpack.py (100%) diff --git a/python/testformat.py b/python/test/test_format.py similarity index 100% rename from python/testformat.py rename to python/test/test_format.py diff --git a/python/test_sequnpack.py b/python/test/test_sequnpack.py similarity index 100% rename from python/test_sequnpack.py rename to python/test/test_sequnpack.py From 04d8fc114b256a80f1fac4e04352d487c47f6768 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sat, 27 Jun 2009 12:04:11 +0900 Subject: [PATCH 0203/1172] Rename test files. --- testformat.py => test/test_format.py | 0 test_sequnpack.py => test/test_sequnpack.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename testformat.py => test/test_format.py (100%) rename test_sequnpack.py => test/test_sequnpack.py (100%) diff --git a/testformat.py b/test/test_format.py similarity index 100% rename from testformat.py rename to test/test_format.py diff --git a/test_sequnpack.py b/test/test_sequnpack.py similarity index 100% rename from test_sequnpack.py rename to test/test_sequnpack.py From 1b07b61c048f749c7a5e505617b0e59974f81077 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 28 Jun 2009 21:24:02 +0900 Subject: [PATCH 0204/1172] Ues more suitable type when packing. --- python/msgpack/_msgpack.pyx | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index c37f8ba..cde5313 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -7,6 +7,8 @@ cdef extern from "Python.h": ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) char* PyString_AsString(object o) + int PyMapping_Check(object o) + int PySequence_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -37,7 +39,7 @@ cdef extern from "pack.h": void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef class Packer: +cdef class Packer(object): """Packer that pack data into strm. strm must have `write(bytes)` method. @@ -99,7 +101,8 @@ cdef class Packer: msgpack_pack_map(&self.pk, len) cdef __pack(self, object o): - cdef long long intval + cdef long long llval + cdef long longval cdef double fval cdef char* rawval @@ -110,11 +113,11 @@ cdef class Packer: elif o is False: msgpack_pack_false(&self.pk) elif isinstance(o, long): - intval = o - msgpack_pack_long_long(&self.pk, intval) + llval = o + msgpack_pack_long_long(&self.pk, llval) elif isinstance(o, int): - intval = o - msgpack_pack_long_long(&self.pk, intval) + longval = o + msgpack_pack_long_long(&self.pk, longval) elif isinstance(o, float): fval = o msgpack_pack_double(&self.pk, fval) @@ -127,12 +130,12 @@ cdef class Packer: rawval = o msgpack_pack_raw(&self.pk, len(o)) msgpack_pack_raw_body(&self.pk, rawval, len(o)) - elif isinstance(o, dict): + elif PyMapping_Check(o): msgpack_pack_map(&self.pk, len(o)) for k,v in o.iteritems(): self.pack(k) self.pack(v) - elif isinstance(o, tuple) or isinstance(o, list): + elif PySequence_Check(o): msgpack_pack_array(&self.pk, len(o)) for v in o: self.pack(v) From 1374bce15f3e6643c49a728a773407994a8207b9 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 28 Jun 2009 21:24:02 +0900 Subject: [PATCH 0205/1172] Ues more suitable type when packing. --- msgpack/_msgpack.pyx | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index c37f8ba..cde5313 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -7,6 +7,8 @@ cdef extern from "Python.h": ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) char* PyString_AsString(object o) + int PyMapping_Check(object o) + int PySequence_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -37,7 +39,7 @@ cdef extern from "pack.h": void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef class Packer: +cdef class Packer(object): """Packer that pack data into strm. strm must have `write(bytes)` method. @@ -99,7 +101,8 @@ cdef class Packer: msgpack_pack_map(&self.pk, len) cdef __pack(self, object o): - cdef long long intval + cdef long long llval + cdef long longval cdef double fval cdef char* rawval @@ -110,11 +113,11 @@ cdef class Packer: elif o is False: msgpack_pack_false(&self.pk) elif isinstance(o, long): - intval = o - msgpack_pack_long_long(&self.pk, intval) + llval = o + msgpack_pack_long_long(&self.pk, llval) elif isinstance(o, int): - intval = o - msgpack_pack_long_long(&self.pk, intval) + longval = o + msgpack_pack_long_long(&self.pk, longval) elif isinstance(o, float): fval = o msgpack_pack_double(&self.pk, fval) @@ -127,12 +130,12 @@ cdef class Packer: rawval = o msgpack_pack_raw(&self.pk, len(o)) msgpack_pack_raw_body(&self.pk, rawval, len(o)) - elif isinstance(o, dict): + elif PyMapping_Check(o): msgpack_pack_map(&self.pk, len(o)) for k,v in o.iteritems(): self.pack(k) self.pack(v) - elif isinstance(o, tuple) or isinstance(o, list): + elif PySequence_Check(o): msgpack_pack_array(&self.pk, len(o)) for v in o: self.pack(v) From 3e396ef146940ada28fc8b154189ae9bb206babc Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 28 Jun 2009 21:24:16 +0900 Subject: [PATCH 0206/1172] Don't use C++. --- python/msgpack/unpack.h | 4 ++-- python/setup.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index dd23130..40058d0 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -19,8 +19,8 @@ #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -struct unpack_user { -}; +typedef struct unpack_user { +} unpack_user; #define msgpack_unpack_struct(name) \ diff --git a/python/setup.py b/python/setup.py index e0e0ac2..faadb33 100755 --- a/python/setup.py +++ b/python/setup.py @@ -10,7 +10,6 @@ version = '0.0.1dev' PACKAGE_ROOT = os.getcwdu() INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') msgpack_mod = Extension('msgpack._msgpack', - language="c++", sources=['msgpack/_msgpack.pyx'], include_dirs=[INCLUDE_PATH]) From 5a87c94333904aea236f646c99c73ce25ba3c74a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 28 Jun 2009 21:24:16 +0900 Subject: [PATCH 0207/1172] Don't use C++. --- msgpack/unpack.h | 4 ++-- setup.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index dd23130..40058d0 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -19,8 +19,8 @@ #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -struct unpack_user { -}; +typedef struct unpack_user { +} unpack_user; #define msgpack_unpack_struct(name) \ diff --git a/setup.py b/setup.py index e0e0ac2..faadb33 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,6 @@ version = '0.0.1dev' PACKAGE_ROOT = os.getcwdu() INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') msgpack_mod = Extension('msgpack._msgpack', - language="c++", sources=['msgpack/_msgpack.pyx'], include_dirs=[INCLUDE_PATH]) From 0b33a634a668a478ee86f2c3d9d29c6be85afa6b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 08:23:27 +0900 Subject: [PATCH 0208/1172] Update test_format. --- python/README | 7 +++ python/test/test_format.py | 110 ++++++++++++++++++++----------------- 2 files changed, 67 insertions(+), 50 deletions(-) diff --git a/python/README b/python/README index 12a27b2..243def2 100644 --- a/python/README +++ b/python/README @@ -29,3 +29,10 @@ MessagePack requires gcc currently. So you need to prepare MinGW GCC. $ python setup.py install -c mingw + +TEST +---- +MessagePack uses nosetest for testing. +Run test with following command: + + $ nosetests test diff --git a/python/test/test_format.py b/python/test/test_format.py index a00123c..009a764 100644 --- a/python/test/test_format.py +++ b/python/test/test_format.py @@ -1,64 +1,74 @@ -from unittest import TestCase, main -from msgpack import packs, unpacks +#!/usr/bin/env python +# coding: utf-8 -class TestFormat(TestCase): - def __check(self, obj, expected_packed): - packed = packs(obj) - self.assertEqual(packed, expected_packed) - unpacked = unpacks(packed) - self.assertEqual(unpacked, obj) +from nose import main +from nose.tools import * +from msgpack import unpacks - def testSimpleValues(self): - self.__check(None, '\xc0') - self.__check(True, '\xc3') - self.__check(False, '\xc2') - self.__check( - [None, False, True], - '\x93\xc0\xc2\xc3' - ) +def check(src, should): + assert_equal(unpacks(src), should) - def testFixnum(self): - self.__check( - [[0,64,127], [-32,-16,-1]], - "\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff" - ) - - def testFixArray(self): - self.__check( - [[],[[None]]], - "\x92\x90\x91\x91\xc0" - ) +def testSimpleValue(): + check("\x93\xc0\xc2\xc3", + [None, False, True]) - def testFixRaw(self): - self.__check( - ["", "a", "bc", "def"], - "\x94\xa0\xa1a\xa2bc\xa3def" - ) - pass +def testFixnum(): + check("\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", + [[0,64,127], [-32,-16,-1]] + ) - def testFixMap(self): - self.__check( - {False: {None: None}, True:{None:{}}}, - "\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80" - ) - pass +def testFixArray(): + check("\x92\x90\x91\x91\xc0", + [[],[[None]]], + ) +def testFixRaw(): + check("\x94\xa0\xa1a\xa2bc\xa3def", + ["", "a", "bc", "def"], + ) -class TestUnpack(TestCase): - def __check(self, packed, obj): - self.assertEqual(unpacks(packed), obj) +def testFixMap(): + check( + "\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", + {False: {None: None}, True:{None:{}}}, + ) - def testuint(self): - self.__check( - "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" - "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" - "\xce\xff\xff\xff\xff", - [0, 128, 255, 0, 32768, 65535, 0, - 2147483648, 4294967295], - ) +def testUnsignedInt(): + check( + "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" + "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" + "\xce\xff\xff\xff\xff", + [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + ) +def testSignedInt(): + check("\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" + "\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" + "\xd2\xff\xff\xff\xff", + [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1]) +def testRaw(): + check("\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" + "\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", + ["", "a", "ab", "", "a", "ab"]) +def testArray(): + check("\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" + "\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02" + "\xc2\xc3", + [[], [None], [False,True], [], [None], [False,True]]) + +def testMap(): + check( + "\x96" + "\xde\x00\x00" + "\xde\x00\x01\xc0\xc2" + "\xde\x00\x02\xc0\xc2\xc3\xc2" + "\xdf\x00\x00\x00\x00" + "\xdf\x00\x00\x00\x01\xc0\xc2" + "\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", + [{}, {None: False}, {True: False, None: False}, {}, + {None: False}, {True: False, None: False}]) if __name__ == '__main__': main() From 9105c3513b2c35e225e4286bec582a5b4dc09f0b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 08:23:27 +0900 Subject: [PATCH 0209/1172] Update test_format. --- README | 7 +++ test/test_format.py | 110 ++++++++++++++++++++++++-------------------- 2 files changed, 67 insertions(+), 50 deletions(-) diff --git a/README b/README index 12a27b2..243def2 100644 --- a/README +++ b/README @@ -29,3 +29,10 @@ MessagePack requires gcc currently. So you need to prepare MinGW GCC. $ python setup.py install -c mingw + +TEST +---- +MessagePack uses nosetest for testing. +Run test with following command: + + $ nosetests test diff --git a/test/test_format.py b/test/test_format.py index a00123c..009a764 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -1,64 +1,74 @@ -from unittest import TestCase, main -from msgpack import packs, unpacks +#!/usr/bin/env python +# coding: utf-8 -class TestFormat(TestCase): - def __check(self, obj, expected_packed): - packed = packs(obj) - self.assertEqual(packed, expected_packed) - unpacked = unpacks(packed) - self.assertEqual(unpacked, obj) +from nose import main +from nose.tools import * +from msgpack import unpacks - def testSimpleValues(self): - self.__check(None, '\xc0') - self.__check(True, '\xc3') - self.__check(False, '\xc2') - self.__check( - [None, False, True], - '\x93\xc0\xc2\xc3' - ) +def check(src, should): + assert_equal(unpacks(src), should) - def testFixnum(self): - self.__check( - [[0,64,127], [-32,-16,-1]], - "\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff" - ) - - def testFixArray(self): - self.__check( - [[],[[None]]], - "\x92\x90\x91\x91\xc0" - ) +def testSimpleValue(): + check("\x93\xc0\xc2\xc3", + [None, False, True]) - def testFixRaw(self): - self.__check( - ["", "a", "bc", "def"], - "\x94\xa0\xa1a\xa2bc\xa3def" - ) - pass +def testFixnum(): + check("\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", + [[0,64,127], [-32,-16,-1]] + ) - def testFixMap(self): - self.__check( - {False: {None: None}, True:{None:{}}}, - "\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80" - ) - pass +def testFixArray(): + check("\x92\x90\x91\x91\xc0", + [[],[[None]]], + ) +def testFixRaw(): + check("\x94\xa0\xa1a\xa2bc\xa3def", + ["", "a", "bc", "def"], + ) -class TestUnpack(TestCase): - def __check(self, packed, obj): - self.assertEqual(unpacks(packed), obj) +def testFixMap(): + check( + "\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", + {False: {None: None}, True:{None:{}}}, + ) - def testuint(self): - self.__check( - "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" - "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" - "\xce\xff\xff\xff\xff", - [0, 128, 255, 0, 32768, 65535, 0, - 2147483648, 4294967295], - ) +def testUnsignedInt(): + check( + "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" + "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" + "\xce\xff\xff\xff\xff", + [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + ) +def testSignedInt(): + check("\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" + "\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" + "\xd2\xff\xff\xff\xff", + [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1]) +def testRaw(): + check("\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" + "\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", + ["", "a", "ab", "", "a", "ab"]) +def testArray(): + check("\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" + "\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02" + "\xc2\xc3", + [[], [None], [False,True], [], [None], [False,True]]) + +def testMap(): + check( + "\x96" + "\xde\x00\x00" + "\xde\x00\x01\xc0\xc2" + "\xde\x00\x02\xc0\xc2\xc3\xc2" + "\xdf\x00\x00\x00\x00" + "\xdf\x00\x00\x00\x01\xc0\xc2" + "\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", + [{}, {None: False}, {True: False, None: False}, {}, + {None: False}, {True: False, None: False}]) if __name__ == '__main__': main() From 257270c1ebc5bb6ac6c60e47130e2a577ffb6714 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 08:23:49 +0900 Subject: [PATCH 0210/1172] Refactor packing code. --- python/msgpack/_msgpack.pyx | 14 +++++++------- python/msgpack/pack.h | 17 ----------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index cde5313..279fdf5 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -31,6 +31,7 @@ cdef extern from "pack.h": void msgpack_pack_nil(msgpack_packer* pk) void msgpack_pack_true(msgpack_packer* pk) void msgpack_pack_false(msgpack_packer* pk) + void msgpack_pack_long(msgpack_packer* pk, long d) void msgpack_pack_long_long(msgpack_packer* pk, long long d) void msgpack_pack_double(msgpack_packer* pk, double d) void msgpack_pack_array(msgpack_packer* pk, size_t l) @@ -60,7 +61,7 @@ cdef class Packer(object): msgpack_packer_init(&self.pk, self, _packer_write) def __del__(self): - free(self.buff); + free(self.buff) def flush(self): """Flash local buffer and output stream if it has 'flush()' method.""" @@ -117,7 +118,7 @@ cdef class Packer(object): msgpack_pack_long_long(&self.pk, llval) elif isinstance(o, int): longval = o - msgpack_pack_long_long(&self.pk, longval) + msgpack_pack_long(&self.pk, longval) elif isinstance(o, float): fval = o msgpack_pack_double(&self.pk, fval) @@ -133,12 +134,12 @@ cdef class Packer(object): elif PyMapping_Check(o): msgpack_pack_map(&self.pk, len(o)) for k,v in o.iteritems(): - self.pack(k) - self.pack(v) + self.__pack(k) + self.__pack(v) elif PySequence_Check(o): msgpack_pack_array(&self.pk, len(o)) for v in o: - self.pack(v) + self.__pack(v) else: # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) @@ -154,7 +155,7 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): if packer.length + l > packer.allocated: if packer.length > 0: packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - if l > 64: + if l > packer.allocated/4: packer.strm.write(PyString_FromStringAndSize(b, l)) packer.length = 0 else: @@ -176,7 +177,6 @@ def packb(object o): buf = StringIO() packer = Packer(buf) packer.pack(o) - packer.flush() return buf.getvalue() packs = packb diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index 9bd6b68..cdac819 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -34,9 +34,6 @@ typedef struct msgpack_packer { static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); -static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); -static inline void msgpack_packer_free(msgpack_packer* pk); - static inline int msgpack_pack_short(msgpack_packer* pk, short d); static inline int msgpack_pack_int(msgpack_packer* pk, int d); static inline int msgpack_pack_long(msgpack_packer* pk, long d); @@ -90,20 +87,6 @@ static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_p pk->callback = callback; } -static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback) -{ - msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer)); - if(!pk) { return NULL; } - msgpack_packer_init(pk, data, callback); - return pk; -} - -static inline void msgpack_packer_free(msgpack_packer* pk) -{ - free(pk); -} - - #ifdef __cplusplus } #endif From 3a6f6626ebf42aae84bfe9a7803bd8ab81185be4 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 08:23:49 +0900 Subject: [PATCH 0211/1172] Refactor packing code. --- msgpack/_msgpack.pyx | 14 +++++++------- msgpack/pack.h | 17 ----------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index cde5313..279fdf5 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -31,6 +31,7 @@ cdef extern from "pack.h": void msgpack_pack_nil(msgpack_packer* pk) void msgpack_pack_true(msgpack_packer* pk) void msgpack_pack_false(msgpack_packer* pk) + void msgpack_pack_long(msgpack_packer* pk, long d) void msgpack_pack_long_long(msgpack_packer* pk, long long d) void msgpack_pack_double(msgpack_packer* pk, double d) void msgpack_pack_array(msgpack_packer* pk, size_t l) @@ -60,7 +61,7 @@ cdef class Packer(object): msgpack_packer_init(&self.pk, self, _packer_write) def __del__(self): - free(self.buff); + free(self.buff) def flush(self): """Flash local buffer and output stream if it has 'flush()' method.""" @@ -117,7 +118,7 @@ cdef class Packer(object): msgpack_pack_long_long(&self.pk, llval) elif isinstance(o, int): longval = o - msgpack_pack_long_long(&self.pk, longval) + msgpack_pack_long(&self.pk, longval) elif isinstance(o, float): fval = o msgpack_pack_double(&self.pk, fval) @@ -133,12 +134,12 @@ cdef class Packer(object): elif PyMapping_Check(o): msgpack_pack_map(&self.pk, len(o)) for k,v in o.iteritems(): - self.pack(k) - self.pack(v) + self.__pack(k) + self.__pack(v) elif PySequence_Check(o): msgpack_pack_array(&self.pk, len(o)) for v in o: - self.pack(v) + self.__pack(v) else: # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) @@ -154,7 +155,7 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): if packer.length + l > packer.allocated: if packer.length > 0: packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - if l > 64: + if l > packer.allocated/4: packer.strm.write(PyString_FromStringAndSize(b, l)) packer.length = 0 else: @@ -176,7 +177,6 @@ def packb(object o): buf = StringIO() packer = Packer(buf) packer.pack(o) - packer.flush() return buf.getvalue() packs = packb diff --git a/msgpack/pack.h b/msgpack/pack.h index 9bd6b68..cdac819 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -34,9 +34,6 @@ typedef struct msgpack_packer { static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); -static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); -static inline void msgpack_packer_free(msgpack_packer* pk); - static inline int msgpack_pack_short(msgpack_packer* pk, short d); static inline int msgpack_pack_int(msgpack_packer* pk, int d); static inline int msgpack_pack_long(msgpack_packer* pk, long d); @@ -90,20 +87,6 @@ static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_p pk->callback = callback; } -static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback) -{ - msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer)); - if(!pk) { return NULL; } - msgpack_packer_init(pk, data, callback); - return pk; -} - -static inline void msgpack_packer_free(msgpack_packer* pk) -{ - free(pk); -} - - #ifdef __cplusplus } #endif From fe2421275d1e0809a9658dbaa5b31fc535bcf32a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 09:31:43 +0900 Subject: [PATCH 0212/1172] Add some test cases. --- python/test/test_case.py | 101 +++++++++++++++++++++++++++++++++++++++ python/test/test_pack.py | 28 +++++++++++ 2 files changed, 129 insertions(+) create mode 100644 python/test/test_case.py create mode 100644 python/test/test_pack.py diff --git a/python/test/test_case.py b/python/test/test_case.py new file mode 100644 index 0000000..3fafd8b --- /dev/null +++ b/python/test/test_case.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * +from msgpack import packs, unpacks + +def check(length, obj): + v = packs(obj) + assert_equal(len(v), length) + assert_equal(unpacks(v), obj) + +def test_1(): + for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1, + -((1<<5)-1), -(1<<5)]: + check(1, o) + +def test_2(): + for o in [1 << 7, (1 << 8) - 1, + -((1<<5)+1), -(1<<7) + ]: + check(2, o) + +def test_3(): + for o in [1 << 8, (1 << 16) - 1, + -((1<<7)+1), -(1<<15)]: + check(3, o) + +def test_5(): + for o in [1 << 16, (1 << 32) - 1, + -((1<<15)+1), -(1<<31)]: + check(5, o) + +def test_9(): + for o in [1 << 32, (1 << 64) - 1, + -((1<<31)+1), -(1<<63), + 1.0, 0.1, -0.1, -1.0]: + check(9, o) + + +def check_raw(overhead, num): + check(num + overhead, " " * num) + +def test_fixraw(): + check_raw(1, 0) + check_raw(1, (1<<5) - 1) + +def test_raw16(): + check_raw(3, 1<<5) + check_raw(3, (1<<16) - 1) + +def test_raw32(): + check_raw(5, 1<<16) + + +def check_array(overhead, num): + check(num + overhead, [None] * num) + +def test_fixarray(): + check_array(1, 0) + check_array(1, (1 << 4) - 1) + +def test_array16(): + check_array(3, 1 << 4) + check_array(3, (1<<16)-1) + +def test_array32(): + check_array(5, (1<<16)) + + +def match(obj, buf): + assert_equal(packs(obj), buf) + assert_equal(unpacks(buf), obj) + +def test_match(): + cases = [ + (None, '\xc0'), + (False, '\xc2'), + (True, '\xc3'), + (0, '\x00'), + (127, '\x7f'), + (128, '\xcc\x80'), + (256, '\xcd\x01\x00'), + (-1, '\xff'), + (-33, '\xd0\xdf'), + (-129, '\xd1\xff\x7f'), + ({1:1}, '\x81\x01\x01'), + (1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"), + ([], '\x90'), + (range(15),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), + (range(16),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), + ({}, '\x80'), + (dict([(x,x) for x in range(15)]), "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"), + (dict([(x,x) for x in range(16)]), "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"), + ] + + for v, p in cases: + match(v, p) + +if __name__ == '__main__': + main() diff --git a/python/test/test_pack.py b/python/test/test_pack.py new file mode 100644 index 0000000..86badb5 --- /dev/null +++ b/python/test/test_pack.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * + +from msgpack import packs, unpacks + +def check(data): + re = unpacks(packs(data)) + assert_equal(re, data) + +def testPack(): + test_data = [ + 0, 1, 127, 128, 255, 256, 65535, 65536, + -1, -32, -33, -128, -129, -32768, -32769, + 1.0, + "", "a", "a"*31, "a"*32, + None, True, False, + [], [[]], [[], None], + {None: 0}, + (1<<23), + ] + for td in test_data: + check(td) + +if __name__ == '__main__': + main() From ca0bda01f116f04f98009f13e1bc9def8200cbb6 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 09:31:43 +0900 Subject: [PATCH 0213/1172] Add some test cases. --- test/test_case.py | 101 ++++++++++++++++++++++++++++++++++++++++++++++ test/test_pack.py | 28 +++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 test/test_case.py create mode 100644 test/test_pack.py diff --git a/test/test_case.py b/test/test_case.py new file mode 100644 index 0000000..3fafd8b --- /dev/null +++ b/test/test_case.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * +from msgpack import packs, unpacks + +def check(length, obj): + v = packs(obj) + assert_equal(len(v), length) + assert_equal(unpacks(v), obj) + +def test_1(): + for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1, + -((1<<5)-1), -(1<<5)]: + check(1, o) + +def test_2(): + for o in [1 << 7, (1 << 8) - 1, + -((1<<5)+1), -(1<<7) + ]: + check(2, o) + +def test_3(): + for o in [1 << 8, (1 << 16) - 1, + -((1<<7)+1), -(1<<15)]: + check(3, o) + +def test_5(): + for o in [1 << 16, (1 << 32) - 1, + -((1<<15)+1), -(1<<31)]: + check(5, o) + +def test_9(): + for o in [1 << 32, (1 << 64) - 1, + -((1<<31)+1), -(1<<63), + 1.0, 0.1, -0.1, -1.0]: + check(9, o) + + +def check_raw(overhead, num): + check(num + overhead, " " * num) + +def test_fixraw(): + check_raw(1, 0) + check_raw(1, (1<<5) - 1) + +def test_raw16(): + check_raw(3, 1<<5) + check_raw(3, (1<<16) - 1) + +def test_raw32(): + check_raw(5, 1<<16) + + +def check_array(overhead, num): + check(num + overhead, [None] * num) + +def test_fixarray(): + check_array(1, 0) + check_array(1, (1 << 4) - 1) + +def test_array16(): + check_array(3, 1 << 4) + check_array(3, (1<<16)-1) + +def test_array32(): + check_array(5, (1<<16)) + + +def match(obj, buf): + assert_equal(packs(obj), buf) + assert_equal(unpacks(buf), obj) + +def test_match(): + cases = [ + (None, '\xc0'), + (False, '\xc2'), + (True, '\xc3'), + (0, '\x00'), + (127, '\x7f'), + (128, '\xcc\x80'), + (256, '\xcd\x01\x00'), + (-1, '\xff'), + (-33, '\xd0\xdf'), + (-129, '\xd1\xff\x7f'), + ({1:1}, '\x81\x01\x01'), + (1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"), + ([], '\x90'), + (range(15),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), + (range(16),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), + ({}, '\x80'), + (dict([(x,x) for x in range(15)]), "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"), + (dict([(x,x) for x in range(16)]), "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"), + ] + + for v, p in cases: + match(v, p) + +if __name__ == '__main__': + main() diff --git a/test/test_pack.py b/test/test_pack.py new file mode 100644 index 0000000..86badb5 --- /dev/null +++ b/test/test_pack.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * + +from msgpack import packs, unpacks + +def check(data): + re = unpacks(packs(data)) + assert_equal(re, data) + +def testPack(): + test_data = [ + 0, 1, 127, 128, 255, 256, 65535, 65536, + -1, -32, -33, -128, -129, -32768, -32769, + 1.0, + "", "a", "a"*31, "a"*32, + None, True, False, + [], [[]], [[], None], + {None: 0}, + (1<<23), + ] + for td in test_data: + check(td) + +if __name__ == '__main__': + main() From 9015bd4ecfe7c30aef47d7c3c6fbc6fd4219eae0 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 10:09:04 +0900 Subject: [PATCH 0214/1172] Fix error on packing unsigned long long. --- python/msgpack/_msgpack.pyx | 34 +++++++++++++++++++++------------- python/test/test_case.py | 2 +- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 279fdf5..e1c497b 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -9,6 +9,8 @@ cdef extern from "Python.h": char* PyString_AsString(object o) int PyMapping_Check(object o) int PySequence_Check(object o) + long long PyLong_AsLongLong(object o) + unsigned long long PyLong_AsUnsignedLongLong(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -27,17 +29,18 @@ cdef extern from "pack.h": msgpack_packer_write callback void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) - void msgpack_pack_int(msgpack_packer* pk, int d) - void msgpack_pack_nil(msgpack_packer* pk) - void msgpack_pack_true(msgpack_packer* pk) - void msgpack_pack_false(msgpack_packer* pk) - void msgpack_pack_long(msgpack_packer* pk, long d) - void msgpack_pack_long_long(msgpack_packer* pk, long long d) - void msgpack_pack_double(msgpack_packer* pk, double d) - void msgpack_pack_array(msgpack_packer* pk, size_t l) - void msgpack_pack_map(msgpack_packer* pk, size_t l) - void msgpack_pack_raw(msgpack_packer* pk, size_t l) - void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) + int msgpack_pack_int(msgpack_packer* pk, int d) + int msgpack_pack_nil(msgpack_packer* pk) + int msgpack_pack_true(msgpack_packer* pk) + int msgpack_pack_false(msgpack_packer* pk) + int msgpack_pack_long(msgpack_packer* pk, long d) + int msgpack_pack_long_long(msgpack_packer* pk, long long d) + int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) + int msgpack_pack_double(msgpack_packer* pk, double d) + int msgpack_pack_array(msgpack_packer* pk, size_t l) + int msgpack_pack_map(msgpack_packer* pk, size_t l) + int msgpack_pack_raw(msgpack_packer* pk, size_t l) + int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) cdef class Packer(object): @@ -103,6 +106,7 @@ cdef class Packer(object): cdef __pack(self, object o): cdef long long llval + cdef unsigned long long ullval cdef long longval cdef double fval cdef char* rawval @@ -114,8 +118,12 @@ cdef class Packer(object): elif o is False: msgpack_pack_false(&self.pk) elif isinstance(o, long): - llval = o - msgpack_pack_long_long(&self.pk, llval) + if o > 0: + ullval = PyLong_AsUnsignedLongLong(o) + msgpack_pack_unsigned_long_long(&self.pk, ullval) + else: + llval = PyLong_AsLongLong(o) + msgpack_pack_long_long(&self.pk, llval) elif isinstance(o, int): longval = o msgpack_pack_long(&self.pk, longval) diff --git a/python/test/test_case.py b/python/test/test_case.py index 3fafd8b..997027a 100644 --- a/python/test/test_case.py +++ b/python/test/test_case.py @@ -7,7 +7,7 @@ from msgpack import packs, unpacks def check(length, obj): v = packs(obj) - assert_equal(len(v), length) + assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) assert_equal(unpacks(v), obj) def test_1(): From f46498bcdfc862f9073ea7f520514cd722b03616 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 10:09:04 +0900 Subject: [PATCH 0215/1172] Fix error on packing unsigned long long. --- msgpack/_msgpack.pyx | 34 +++++++++++++++++++++------------- test/test_case.py | 2 +- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 279fdf5..e1c497b 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -9,6 +9,8 @@ cdef extern from "Python.h": char* PyString_AsString(object o) int PyMapping_Check(object o) int PySequence_Check(object o) + long long PyLong_AsLongLong(object o) + unsigned long long PyLong_AsUnsignedLongLong(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -27,17 +29,18 @@ cdef extern from "pack.h": msgpack_packer_write callback void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) - void msgpack_pack_int(msgpack_packer* pk, int d) - void msgpack_pack_nil(msgpack_packer* pk) - void msgpack_pack_true(msgpack_packer* pk) - void msgpack_pack_false(msgpack_packer* pk) - void msgpack_pack_long(msgpack_packer* pk, long d) - void msgpack_pack_long_long(msgpack_packer* pk, long long d) - void msgpack_pack_double(msgpack_packer* pk, double d) - void msgpack_pack_array(msgpack_packer* pk, size_t l) - void msgpack_pack_map(msgpack_packer* pk, size_t l) - void msgpack_pack_raw(msgpack_packer* pk, size_t l) - void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) + int msgpack_pack_int(msgpack_packer* pk, int d) + int msgpack_pack_nil(msgpack_packer* pk) + int msgpack_pack_true(msgpack_packer* pk) + int msgpack_pack_false(msgpack_packer* pk) + int msgpack_pack_long(msgpack_packer* pk, long d) + int msgpack_pack_long_long(msgpack_packer* pk, long long d) + int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) + int msgpack_pack_double(msgpack_packer* pk, double d) + int msgpack_pack_array(msgpack_packer* pk, size_t l) + int msgpack_pack_map(msgpack_packer* pk, size_t l) + int msgpack_pack_raw(msgpack_packer* pk, size_t l) + int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) cdef class Packer(object): @@ -103,6 +106,7 @@ cdef class Packer(object): cdef __pack(self, object o): cdef long long llval + cdef unsigned long long ullval cdef long longval cdef double fval cdef char* rawval @@ -114,8 +118,12 @@ cdef class Packer(object): elif o is False: msgpack_pack_false(&self.pk) elif isinstance(o, long): - llval = o - msgpack_pack_long_long(&self.pk, llval) + if o > 0: + ullval = PyLong_AsUnsignedLongLong(o) + msgpack_pack_unsigned_long_long(&self.pk, ullval) + else: + llval = PyLong_AsLongLong(o) + msgpack_pack_long_long(&self.pk, llval) elif isinstance(o, int): longval = o msgpack_pack_long(&self.pk, longval) diff --git a/test/test_case.py b/test/test_case.py index 3fafd8b..997027a 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -7,7 +7,7 @@ from msgpack import packs, unpacks def check(length, obj): v = packs(obj) - assert_equal(len(v), length) + assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) assert_equal(unpacks(v), obj) def test_1(): From b5010c71a9810515db1aa82d2e7eebf2ee4f474f Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 11:21:28 +0900 Subject: [PATCH 0216/1172] Fix tests. --- python/Makefile | 7 +++++++ python/test/test_case.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 python/Makefile diff --git a/python/Makefile b/python/Makefile new file mode 100644 index 0000000..5f16fae --- /dev/null +++ b/python/Makefile @@ -0,0 +1,7 @@ +all: + python setup.py build + python setup.py sdist + +.PHONY: test +test: + nosetests test diff --git a/python/test/test_case.py b/python/test/test_case.py index 997027a..d754fb0 100644 --- a/python/test/test_case.py +++ b/python/test/test_case.py @@ -90,8 +90,8 @@ def test_match(): (range(15),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), (range(16),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), ({}, '\x80'), - (dict([(x,x) for x in range(15)]), "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"), - (dict([(x,x) for x in range(16)]), "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"), + (dict([(x,x) for x in range(15)]), '\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'), + (dict([(x,x) for x in range(16)]), '\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'), ] for v, p in cases: From 6201e623e29b4162701a9ccdce07af893baaaa5e Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 11:21:28 +0900 Subject: [PATCH 0217/1172] Fix tests. --- Makefile | 7 +++++++ test/test_case.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5f16fae --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +all: + python setup.py build + python setup.py sdist + +.PHONY: test +test: + nosetests test diff --git a/test/test_case.py b/test/test_case.py index 997027a..d754fb0 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -90,8 +90,8 @@ def test_match(): (range(15),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), (range(16),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), ({}, '\x80'), - (dict([(x,x) for x in range(15)]), "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"), - (dict([(x,x) for x in range(16)]), "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"), + (dict([(x,x) for x in range(15)]), '\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'), + (dict([(x,x) for x in range(16)]), '\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'), ] for v, p in cases: From d4317fdc853c7dbe3bc57718dc009cb8b0a2b351 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 30 Jun 2009 23:03:33 +0900 Subject: [PATCH 0218/1172] Some optimization on packing. --- python/Makefile | 1 + python/msgpack/_msgpack.pyx | 69 +++++++++++++------------------------ python/msgpack/pack.h | 29 +++++++++------- 3 files changed, 40 insertions(+), 59 deletions(-) diff --git a/python/Makefile b/python/Makefile index 5f16fae..d90789a 100644 --- a/python/Makefile +++ b/python/Makefile @@ -1,4 +1,5 @@ all: + python setup.py build_ext -i -f python setup.py build python setup.py sdist diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index e1c497b..26222aa 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -1,16 +1,23 @@ # coding: utf-8 -from cStringIO import StringIO +import cStringIO cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + PyObject* Py_True + PyObject* Py_False char* PyString_AsString(object o) int PyMapping_Check(object o) int PySequence_Check(object o) long long PyLong_AsLongLong(object o) unsigned long long PyLong_AsUnsignedLongLong(object o) + int PyLong_Check(object o) + int PyInt_Check(object o) + int PyFloat_Check(object o) + int PyString_Check(object o) + int PyUnicode_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -22,13 +29,9 @@ cdef extern from "string.h": void* memmove(char* dst, char* src, size_t size) cdef extern from "pack.h": - ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) - struct msgpack_packer: - void *data - msgpack_packer_write callback + PyObject* writer - void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) int msgpack_pack_int(msgpack_packer* pk, int d) int msgpack_pack_nil(msgpack_packer* pk) int msgpack_pack_true(msgpack_packer* pk) @@ -49,28 +52,17 @@ cdef class Packer(object): strm must have `write(bytes)` method. size specifies local buffer size. """ - cdef char* buff - cdef unsigned int length - cdef unsigned int allocated cdef msgpack_packer pk cdef object strm + cdef object writer - def __init__(self, strm, int size=4*1024): - self.strm = strm - self.buff = malloc(size) - self.allocated = size - self.length = 0 - - msgpack_packer_init(&self.pk, self, _packer_write) - - def __del__(self): - free(self.buff) + def __init__(self, strm_, int size=4*1024): + self.strm = strm_ + self.writer = strm_.write + self.pk.writer = self.writer def flush(self): """Flash local buffer and output stream if it has 'flush()' method.""" - if self.length > 0: - self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - self.length = 0 if hasattr(self.strm, 'flush'): self.strm.flush() @@ -113,28 +105,28 @@ cdef class Packer(object): if o is None: msgpack_pack_nil(&self.pk) - elif o is True: + elif o == Py_True: msgpack_pack_true(&self.pk) - elif o is False: + elif o == Py_False: msgpack_pack_false(&self.pk) - elif isinstance(o, long): + elif PyLong_Check(o): if o > 0: ullval = PyLong_AsUnsignedLongLong(o) msgpack_pack_unsigned_long_long(&self.pk, ullval) else: llval = PyLong_AsLongLong(o) msgpack_pack_long_long(&self.pk, llval) - elif isinstance(o, int): + elif PyInt_Check(o): longval = o msgpack_pack_long(&self.pk, longval) - elif isinstance(o, float): + elif PyFloat_Check(o): fval = o msgpack_pack_double(&self.pk, fval) - elif isinstance(o, str): + elif PyString_Check(o): rawval = o msgpack_pack_raw(&self.pk, len(o)) msgpack_pack_raw_body(&self.pk, rawval, len(o)) - elif isinstance(o, unicode): + elif PyUnicode_Check(o): o = o.encode('utf-8') rawval = o msgpack_pack_raw(&self.pk, len(o)) @@ -152,28 +144,13 @@ cdef class Packer(object): # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) - def pack(self, obj, flush=True): + def pack(self, object obj, flush=True): self.__pack(obj) if flush: self.flush() close = flush -cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - if packer.length + l > packer.allocated: - if packer.length > 0: - packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - if l > packer.allocated/4: - packer.strm.write(PyString_FromStringAndSize(b, l)) - packer.length = 0 - else: - memcpy(packer.buff, b, l) - packer.length = l - else: - memcpy(packer.buff + packer.length, b, l) - packer.length += l - return 0 - def pack(object o, object stream): u"""pack o and write to stream).""" packer = Packer(stream) @@ -182,7 +159,7 @@ def pack(object o, object stream): def packb(object o): u"""pack o and return packed bytes.""" - buf = StringIO() + buf = cStringIO.StringIO() packer = Packer(buf) packer.pack(o) return buf.getvalue() diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index cdac819..ea97601 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -24,15 +24,11 @@ extern "C" { #endif - -typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len); - typedef struct msgpack_packer { - void* data; - msgpack_packer_write callback; + PyObject* writer; } msgpack_packer; -static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); +typedef struct Packer Packer; static inline int msgpack_pack_short(msgpack_packer* pk, short d); static inline int msgpack_pack_int(msgpack_packer* pk, int d); @@ -66,7 +62,20 @@ static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n); static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); +static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) +{ + PyObject *buf, *ret; + buf = PyBuffer_FromMemory((void*)data, l); + //buf = PyString_FromStringAndSize(data, l); + if (buf == NULL) return -1; + + ret = PyObject_CallFunctionObjArgs(pk->writer, buf, NULL); + Py_DECREF(buf); + if (ret == NULL) return -1; + Py_DECREF(ret); + return 0; +} #define msgpack_pack_inline_func(name) \ static inline int msgpack_pack ## name @@ -77,16 +86,10 @@ static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_ #define msgpack_pack_user msgpack_packer* #define msgpack_pack_append_buffer(user, buf, len) \ - return (*(user)->callback)((user)->data, (const char*)buf, len) + return msgpack_pack_write(user, (const char*)buf, len) #include "pack_template.h" -static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) -{ - pk->data = data; - pk->callback = callback; -} - #ifdef __cplusplus } #endif From f7b3ef18b0ca44a14ba39087f04cad14433542a0 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 30 Jun 2009 23:03:33 +0900 Subject: [PATCH 0219/1172] Some optimization on packing. --- Makefile | 1 + msgpack/_msgpack.pyx | 69 +++++++++++++++----------------------------- msgpack/pack.h | 29 ++++++++++--------- 3 files changed, 40 insertions(+), 59 deletions(-) diff --git a/Makefile b/Makefile index 5f16fae..d90789a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ all: + python setup.py build_ext -i -f python setup.py build python setup.py sdist diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index e1c497b..26222aa 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -1,16 +1,23 @@ # coding: utf-8 -from cStringIO import StringIO +import cStringIO cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + PyObject* Py_True + PyObject* Py_False char* PyString_AsString(object o) int PyMapping_Check(object o) int PySequence_Check(object o) long long PyLong_AsLongLong(object o) unsigned long long PyLong_AsUnsignedLongLong(object o) + int PyLong_Check(object o) + int PyInt_Check(object o) + int PyFloat_Check(object o) + int PyString_Check(object o) + int PyUnicode_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -22,13 +29,9 @@ cdef extern from "string.h": void* memmove(char* dst, char* src, size_t size) cdef extern from "pack.h": - ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) - struct msgpack_packer: - void *data - msgpack_packer_write callback + PyObject* writer - void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) int msgpack_pack_int(msgpack_packer* pk, int d) int msgpack_pack_nil(msgpack_packer* pk) int msgpack_pack_true(msgpack_packer* pk) @@ -49,28 +52,17 @@ cdef class Packer(object): strm must have `write(bytes)` method. size specifies local buffer size. """ - cdef char* buff - cdef unsigned int length - cdef unsigned int allocated cdef msgpack_packer pk cdef object strm + cdef object writer - def __init__(self, strm, int size=4*1024): - self.strm = strm - self.buff = malloc(size) - self.allocated = size - self.length = 0 - - msgpack_packer_init(&self.pk, self, _packer_write) - - def __del__(self): - free(self.buff) + def __init__(self, strm_, int size=4*1024): + self.strm = strm_ + self.writer = strm_.write + self.pk.writer = self.writer def flush(self): """Flash local buffer and output stream if it has 'flush()' method.""" - if self.length > 0: - self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - self.length = 0 if hasattr(self.strm, 'flush'): self.strm.flush() @@ -113,28 +105,28 @@ cdef class Packer(object): if o is None: msgpack_pack_nil(&self.pk) - elif o is True: + elif o == Py_True: msgpack_pack_true(&self.pk) - elif o is False: + elif o == Py_False: msgpack_pack_false(&self.pk) - elif isinstance(o, long): + elif PyLong_Check(o): if o > 0: ullval = PyLong_AsUnsignedLongLong(o) msgpack_pack_unsigned_long_long(&self.pk, ullval) else: llval = PyLong_AsLongLong(o) msgpack_pack_long_long(&self.pk, llval) - elif isinstance(o, int): + elif PyInt_Check(o): longval = o msgpack_pack_long(&self.pk, longval) - elif isinstance(o, float): + elif PyFloat_Check(o): fval = o msgpack_pack_double(&self.pk, fval) - elif isinstance(o, str): + elif PyString_Check(o): rawval = o msgpack_pack_raw(&self.pk, len(o)) msgpack_pack_raw_body(&self.pk, rawval, len(o)) - elif isinstance(o, unicode): + elif PyUnicode_Check(o): o = o.encode('utf-8') rawval = o msgpack_pack_raw(&self.pk, len(o)) @@ -152,28 +144,13 @@ cdef class Packer(object): # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) - def pack(self, obj, flush=True): + def pack(self, object obj, flush=True): self.__pack(obj) if flush: self.flush() close = flush -cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - if packer.length + l > packer.allocated: - if packer.length > 0: - packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - if l > packer.allocated/4: - packer.strm.write(PyString_FromStringAndSize(b, l)) - packer.length = 0 - else: - memcpy(packer.buff, b, l) - packer.length = l - else: - memcpy(packer.buff + packer.length, b, l) - packer.length += l - return 0 - def pack(object o, object stream): u"""pack o and write to stream).""" packer = Packer(stream) @@ -182,7 +159,7 @@ def pack(object o, object stream): def packb(object o): u"""pack o and return packed bytes.""" - buf = StringIO() + buf = cStringIO.StringIO() packer = Packer(buf) packer.pack(o) return buf.getvalue() diff --git a/msgpack/pack.h b/msgpack/pack.h index cdac819..ea97601 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -24,15 +24,11 @@ extern "C" { #endif - -typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len); - typedef struct msgpack_packer { - void* data; - msgpack_packer_write callback; + PyObject* writer; } msgpack_packer; -static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); +typedef struct Packer Packer; static inline int msgpack_pack_short(msgpack_packer* pk, short d); static inline int msgpack_pack_int(msgpack_packer* pk, int d); @@ -66,7 +62,20 @@ static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n); static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); +static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) +{ + PyObject *buf, *ret; + buf = PyBuffer_FromMemory((void*)data, l); + //buf = PyString_FromStringAndSize(data, l); + if (buf == NULL) return -1; + + ret = PyObject_CallFunctionObjArgs(pk->writer, buf, NULL); + Py_DECREF(buf); + if (ret == NULL) return -1; + Py_DECREF(ret); + return 0; +} #define msgpack_pack_inline_func(name) \ static inline int msgpack_pack ## name @@ -77,16 +86,10 @@ static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_ #define msgpack_pack_user msgpack_packer* #define msgpack_pack_append_buffer(user, buf, len) \ - return (*(user)->callback)((user)->data, (const char*)buf, len) + return msgpack_pack_write(user, (const char*)buf, len) #include "pack_template.h" -static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) -{ - pk->data = data; - pk->callback = callback; -} - #ifdef __cplusplus } #endif From 03942a1b9020b00bf47e93b7d5ec606e8160c054 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 1 Jul 2009 00:57:46 +0900 Subject: [PATCH 0220/1172] Major speedup on packing. --- python/msgpack/_msgpack.pyx | 60 +++++++++++++------------------------ python/msgpack/pack.h | 25 ++++++++++------ 2 files changed, 37 insertions(+), 48 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 26222aa..9d766c5 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -18,6 +18,7 @@ cdef extern from "Python.h": int PyFloat_Check(object o) int PyString_Check(object o) int PyUnicode_Check(object o) + object PyBuffer_FromMemory(const_char_ptr b, Py_ssize_t len) cdef extern from "stdlib.h": void* malloc(size_t) @@ -30,7 +31,9 @@ cdef extern from "string.h": cdef extern from "pack.h": struct msgpack_packer: - PyObject* writer + char* buf + size_t length + size_t buf_size int msgpack_pack_int(msgpack_packer* pk, int d) int msgpack_pack_nil(msgpack_packer* pk) @@ -56,45 +59,16 @@ cdef class Packer(object): cdef object strm cdef object writer - def __init__(self, strm_, int size=4*1024): - self.strm = strm_ - self.writer = strm_.write - self.pk.writer = self.writer + def __init__(self, strm): + self.strm = strm - def flush(self): - """Flash local buffer and output stream if it has 'flush()' method.""" - if hasattr(self.strm, 'flush'): - self.strm.flush() + cdef int buf_size = 1024*1024 + self.pk.buf = malloc(buf_size); + self.pk.buf_size = buf_size + self.pk.length = 0 - def pack_list(self, len): - """Start packing sequential objects. - - Example: - - packer.pack_list(2) - packer.pack('foo') - packer.pack('bar') - - This is same to: - - packer.pack(['foo', 'bar']) - """ - msgpack_pack_array(&self.pk, len) - - def pack_dict(self, len): - """Start packing key-value objects. - - Example: - - packer.pack_dict(1) - packer.pack('foo') - packer.pack('bar') - - This is same to: - - packer.pack({'foo': 'bar'}) - """ - msgpack_pack_map(&self.pk, len) + def __del__(self): + free(self.pk.buf); cdef __pack(self, object o): cdef long long llval @@ -144,11 +118,19 @@ cdef class Packer(object): # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) - def pack(self, object obj, flush=True): + def pack(self, object obj, object flush=True): self.__pack(obj) + buf = PyBuffer_FromMemory(self.pk.buf, self.pk.length) + self.pk.length = 0 + self.strm.write(buf) if flush: self.flush() + def flush(self): + """Flash local buffer and output stream if it has 'flush()' method.""" + if hasattr(self.strm, 'flush'): + self.strm.flush() + close = flush def pack(object o, object stream): diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index ea97601..d7e0867 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -25,7 +25,9 @@ extern "C" { #endif typedef struct msgpack_packer { - PyObject* writer; + char *buf; + size_t length; + size_t buf_size; } msgpack_packer; typedef struct Packer Packer; @@ -64,16 +66,21 @@ static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) { - PyObject *buf, *ret; + char* buf = pk->buf; + size_t bs = pk->buf_size; + size_t len = pk->length; - buf = PyBuffer_FromMemory((void*)data, l); - //buf = PyString_FromStringAndSize(data, l); - if (buf == NULL) return -1; + if (len + l > bs) { + bs = (len + l) * 2; + buf = realloc(pk->buf, bs); + if (!buf) return -1; + } + memcpy(buf + len, data, l); + len += l; - ret = PyObject_CallFunctionObjArgs(pk->writer, buf, NULL); - Py_DECREF(buf); - if (ret == NULL) return -1; - Py_DECREF(ret); + pk->buf = buf; + pk->buf_size = bs; + pk->length = len; return 0; } From 17cbfa66ae0d6171b0e29699459a3dfbf30e06f2 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 1 Jul 2009 00:57:46 +0900 Subject: [PATCH 0221/1172] Major speedup on packing. --- msgpack/_msgpack.pyx | 60 ++++++++++++++++---------------------------- msgpack/pack.h | 25 +++++++++++------- 2 files changed, 37 insertions(+), 48 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 26222aa..9d766c5 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -18,6 +18,7 @@ cdef extern from "Python.h": int PyFloat_Check(object o) int PyString_Check(object o) int PyUnicode_Check(object o) + object PyBuffer_FromMemory(const_char_ptr b, Py_ssize_t len) cdef extern from "stdlib.h": void* malloc(size_t) @@ -30,7 +31,9 @@ cdef extern from "string.h": cdef extern from "pack.h": struct msgpack_packer: - PyObject* writer + char* buf + size_t length + size_t buf_size int msgpack_pack_int(msgpack_packer* pk, int d) int msgpack_pack_nil(msgpack_packer* pk) @@ -56,45 +59,16 @@ cdef class Packer(object): cdef object strm cdef object writer - def __init__(self, strm_, int size=4*1024): - self.strm = strm_ - self.writer = strm_.write - self.pk.writer = self.writer + def __init__(self, strm): + self.strm = strm - def flush(self): - """Flash local buffer and output stream if it has 'flush()' method.""" - if hasattr(self.strm, 'flush'): - self.strm.flush() + cdef int buf_size = 1024*1024 + self.pk.buf = malloc(buf_size); + self.pk.buf_size = buf_size + self.pk.length = 0 - def pack_list(self, len): - """Start packing sequential objects. - - Example: - - packer.pack_list(2) - packer.pack('foo') - packer.pack('bar') - - This is same to: - - packer.pack(['foo', 'bar']) - """ - msgpack_pack_array(&self.pk, len) - - def pack_dict(self, len): - """Start packing key-value objects. - - Example: - - packer.pack_dict(1) - packer.pack('foo') - packer.pack('bar') - - This is same to: - - packer.pack({'foo': 'bar'}) - """ - msgpack_pack_map(&self.pk, len) + def __del__(self): + free(self.pk.buf); cdef __pack(self, object o): cdef long long llval @@ -144,11 +118,19 @@ cdef class Packer(object): # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) - def pack(self, object obj, flush=True): + def pack(self, object obj, object flush=True): self.__pack(obj) + buf = PyBuffer_FromMemory(self.pk.buf, self.pk.length) + self.pk.length = 0 + self.strm.write(buf) if flush: self.flush() + def flush(self): + """Flash local buffer and output stream if it has 'flush()' method.""" + if hasattr(self.strm, 'flush'): + self.strm.flush() + close = flush def pack(object o, object stream): diff --git a/msgpack/pack.h b/msgpack/pack.h index ea97601..d7e0867 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -25,7 +25,9 @@ extern "C" { #endif typedef struct msgpack_packer { - PyObject* writer; + char *buf; + size_t length; + size_t buf_size; } msgpack_packer; typedef struct Packer Packer; @@ -64,16 +66,21 @@ static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) { - PyObject *buf, *ret; + char* buf = pk->buf; + size_t bs = pk->buf_size; + size_t len = pk->length; - buf = PyBuffer_FromMemory((void*)data, l); - //buf = PyString_FromStringAndSize(data, l); - if (buf == NULL) return -1; + if (len + l > bs) { + bs = (len + l) * 2; + buf = realloc(pk->buf, bs); + if (!buf) return -1; + } + memcpy(buf + len, data, l); + len += l; - ret = PyObject_CallFunctionObjArgs(pk->writer, buf, NULL); - Py_DECREF(buf); - if (ret == NULL) return -1; - Py_DECREF(ret); + pk->buf = buf; + pk->buf_size = bs; + pk->length = len; return 0; } From a95e8c790c11927708f29a1640492530955369f7 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 1 Jul 2009 17:45:18 +0900 Subject: [PATCH 0222/1172] this module requires perl 5.8 --- perl/lib/Data/MessagePack.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index d26bf9e..7b5604d 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -2,6 +2,7 @@ package Data::MessagePack; use strict; use warnings; use XSLoader; +use 5.008001; our $VERSION = 0.01; From c6496ddf1334d4dc037cff390c5c227c050bf20d Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 1 Jul 2009 18:08:39 +0900 Subject: [PATCH 0223/1172] added .gitignore file --- perl/.gitignore | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 perl/.gitignore diff --git a/perl/.gitignore b/perl/.gitignore new file mode 100644 index 0000000..53c9b78 --- /dev/null +++ b/perl/.gitignore @@ -0,0 +1,11 @@ +META.yml +Makefile +Makefile.old +MessagePack.bs +MessagePack.o +blib/ +inc/ +msgpack/ +pack.o +pm_to_blib +unpack.o From 37972dfe3ea0adbd9c5f9ffd8898ab3274ec72ba Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 1 Jul 2009 18:20:14 +0900 Subject: [PATCH 0224/1172] copy files into this dir. --- perl/Makefile.PL | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index f60e125..167bebe 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -6,7 +6,7 @@ perl_version '5.008005'; license 'perl'; can_cc or die "This module requires a C compiler"; -my $ccflags = '-I../ '; +my $ccflags = '.'; makemaker_args( OBJECT => '$(O_FILES)', @@ -26,6 +26,15 @@ makemaker_args( tests 't/*.t'; author_tests('xt'); +# copy modules +if ($Module::Install::AUTHOR && -d File::Spec->catfile('..', 'msgpack')) { + mkdir 'msgpack' unless -d 'msgpack'; + require File::Copy; + for my $src (<../msgpack/*.h>) { + File::Copy::copy($src, 'msgpack/') or die "copy failed: $!"; + } +} + auto_set_repository; build_requires 'Test::More'; use_test_base; From d72bef9f9ac7d26e54e0778062d575e887ff3df4 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 1 Jul 2009 18:33:56 +0900 Subject: [PATCH 0225/1172] added MANIFEST.SKIP files --- perl/MANIFEST.SKIP | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 perl/MANIFEST.SKIP diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP new file mode 100644 index 0000000..366a1e8 --- /dev/null +++ b/perl/MANIFEST.SKIP @@ -0,0 +1,21 @@ +\bRCS\b +\bCVS\b +^MANIFEST\. +^Makefile$ +~$ +^# +\.old$ +^blib/ +^pm_to_blib +^MakeMaker-\d +\.gz$ +\.cvsignore +^t/9\d_.*\.t +^t/perlcritic +^tools/ +\.svn/ +^[^/]+\.yaml$ +^[^/]+\.pl$ +^\.shipit$ +^\.git/ +\.sw[pon]$ From d86b90e66344637a31b9def7c8838dd5dd9ec992 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 1 Jul 2009 18:34:11 +0900 Subject: [PATCH 0226/1172] added Changes file --- perl/Changes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 perl/Changes diff --git a/perl/Changes b/perl/Changes new file mode 100644 index 0000000..18041d4 --- /dev/null +++ b/perl/Changes @@ -0,0 +1,3 @@ +0.01 + + - initial release to CPAN From 7c276241610ac7a2920e399346a7690038adaed9 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 1 Jul 2009 18:34:27 +0900 Subject: [PATCH 0227/1172] added .shipit file --- perl/.shipit | 1 + 1 file changed, 1 insertion(+) create mode 100644 perl/.shipit diff --git a/perl/.shipit b/perl/.shipit new file mode 100644 index 0000000..8731dce --- /dev/null +++ b/perl/.shipit @@ -0,0 +1 @@ +steps = FindVersion, ChangeVersion, CheckChangeLog, DistTest, Commit, Tag, MakeDist, UploadCPAN From c2a63b2c54f49c2ab92c5247aa76c02414404b7c Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 1 Jul 2009 18:40:14 +0900 Subject: [PATCH 0228/1172] Checking in changes prior to tagging of version 0.01. Changelog diff is: --- perl/.gitignore | 1 + perl/.shipit | 1 + perl/MANIFEST.SKIP | 1 + perl/lib/Data/MessagePack.pm | 2 +- 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/perl/.gitignore b/perl/.gitignore index 53c9b78..a9096b6 100644 --- a/perl/.gitignore +++ b/perl/.gitignore @@ -9,3 +9,4 @@ msgpack/ pack.o pm_to_blib unpack.o +MANIFEST diff --git a/perl/.shipit b/perl/.shipit index 8731dce..f9997c8 100644 --- a/perl/.shipit +++ b/perl/.shipit @@ -1 +1,2 @@ steps = FindVersion, ChangeVersion, CheckChangeLog, DistTest, Commit, Tag, MakeDist, UploadCPAN +git.tagpattern = perl-%v diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP index 366a1e8..1b268ef 100644 --- a/perl/MANIFEST.SKIP +++ b/perl/MANIFEST.SKIP @@ -19,3 +19,4 @@ ^\.shipit$ ^\.git/ \.sw[pon]$ +^\.gitignore$ diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 7b5604d..cb3f73a 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = 0.01; +our $VERSION = '0.01'; XSLoader::load(__PACKAGE__, $VERSION); From 78db826a75d9f091f2175ace5f2cd9cb81c0f115 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 1 Jul 2009 20:55:24 +0900 Subject: [PATCH 0229/1172] Fix memory leak. Remove stream packing feature. Add errorcheck in packing. --- python/msgpack/_msgpack.pyx | 135 +++++++++++++++++++----------------- python/msgpack/pack.h | 2 +- 2 files changed, 73 insertions(+), 64 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 9d766c5..c455394 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -5,20 +5,22 @@ import cStringIO cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject + cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) - PyObject* Py_True - PyObject* Py_False - char* PyString_AsString(object o) - int PyMapping_Check(object o) - int PySequence_Check(object o) - long long PyLong_AsLongLong(object o) - unsigned long long PyLong_AsUnsignedLongLong(object o) - int PyLong_Check(object o) - int PyInt_Check(object o) - int PyFloat_Check(object o) - int PyString_Check(object o) - int PyUnicode_Check(object o) - object PyBuffer_FromMemory(const_char_ptr b, Py_ssize_t len) + cdef PyObject* Py_True + cdef PyObject* Py_False + + cdef char* PyString_AsString(object o) + cdef long long PyLong_AsLongLong(object o) + cdef unsigned long long PyLong_AsUnsignedLongLong(object o) + + cdef int PyMapping_Check(object o) + cdef int PySequence_Check(object o) + cdef int PyLong_Check(object o) + cdef int PyInt_Check(object o) + cdef int PyFloat_Check(object o) + cdef int PyString_Check(object o) + cdef int PyUnicode_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -50,101 +52,101 @@ cdef extern from "pack.h": cdef class Packer(object): - """Packer that pack data into strm. + """MessagePack Packer + + usage: - strm must have `write(bytes)` method. - size specifies local buffer size. + packer = Packer() + astream.write(packer.pack(a)) + astream.write(packer.pack(b)) """ cdef msgpack_packer pk - cdef object strm - cdef object writer - - def __init__(self, strm): - self.strm = strm + def __cinit__(self): cdef int buf_size = 1024*1024 self.pk.buf = malloc(buf_size); self.pk.buf_size = buf_size self.pk.length = 0 - def __del__(self): + def __dealloc__(self): free(self.pk.buf); - cdef __pack(self, object o): + cdef int __pack(self, object o): cdef long long llval cdef unsigned long long ullval cdef long longval cdef double fval cdef char* rawval + cdef int ret if o is None: - msgpack_pack_nil(&self.pk) + ret = msgpack_pack_nil(&self.pk) elif o == Py_True: - msgpack_pack_true(&self.pk) + ret = msgpack_pack_true(&self.pk) elif o == Py_False: - msgpack_pack_false(&self.pk) + ret = msgpack_pack_false(&self.pk) elif PyLong_Check(o): if o > 0: ullval = PyLong_AsUnsignedLongLong(o) - msgpack_pack_unsigned_long_long(&self.pk, ullval) + ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) else: llval = PyLong_AsLongLong(o) - msgpack_pack_long_long(&self.pk, llval) + ret = msgpack_pack_long_long(&self.pk, llval) elif PyInt_Check(o): longval = o - msgpack_pack_long(&self.pk, longval) + ret = msgpack_pack_long(&self.pk, longval) elif PyFloat_Check(o): fval = o - msgpack_pack_double(&self.pk, fval) + ret = msgpack_pack_double(&self.pk, fval) elif PyString_Check(o): rawval = o - msgpack_pack_raw(&self.pk, len(o)) - msgpack_pack_raw_body(&self.pk, rawval, len(o)) + ret = msgpack_pack_raw(&self.pk, len(o)) + if ret == 0: + ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyUnicode_Check(o): o = o.encode('utf-8') rawval = o - msgpack_pack_raw(&self.pk, len(o)) - msgpack_pack_raw_body(&self.pk, rawval, len(o)) + ret = msgpack_pack_raw(&self.pk, len(o)) + if ret == 0: + ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyMapping_Check(o): - msgpack_pack_map(&self.pk, len(o)) - for k,v in o.iteritems(): - self.__pack(k) - self.__pack(v) + ret = msgpack_pack_map(&self.pk, len(o)) + if ret == 0: + for k,v in o.iteritems(): + ret = self.__pack(k) + if ret != 0: break + ret = self.__pack(v) + if ret != 0: break elif PySequence_Check(o): - msgpack_pack_array(&self.pk, len(o)) - for v in o: - self.__pack(v) + ret = msgpack_pack_array(&self.pk, len(o)) + if ret == 0: + for v in o: + ret = self.__pack(v) + if ret != 0: break else: # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) + return ret - def pack(self, object obj, object flush=True): - self.__pack(obj) - buf = PyBuffer_FromMemory(self.pk.buf, self.pk.length) + def pack(self, object obj): + cdef int ret + ret = self.__pack(obj) + if ret: + raise TypeError + buf = PyString_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 - self.strm.write(buf) - if flush: - self.flush() + return buf - def flush(self): - """Flash local buffer and output stream if it has 'flush()' method.""" - if hasattr(self.strm, 'flush'): - self.strm.flush() - - close = flush def pack(object o, object stream): - u"""pack o and write to stream).""" - packer = Packer(stream) - packer.pack(o) - packer.flush() + """pack a object `o` and write it to stream).""" + packer = Packer() + stream.write(packer.pack(o)) def packb(object o): - u"""pack o and return packed bytes.""" - buf = cStringIO.StringIO() - packer = Packer(buf) - packer.pack(o) - return buf.getvalue() + """pack o and return packed bytes.""" + packer = Packer() + return packer.pack(o) packs = packb @@ -222,7 +224,14 @@ cdef class Unpacker(object): cdef int read_size cdef object waiting_bytes - def __init__(self, file_like=None, int read_size=4096): + def __cinit__(self): + self.buf = NULL + + def __dealloc__(self): + if self.buf: + free(self.buf); + + def __init__(self, file_like=None, int read_size=1024*1024): self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index d7e0867..58f021e 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -72,7 +72,7 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ if (len + l > bs) { bs = (len + l) * 2; - buf = realloc(pk->buf, bs); + buf = realloc(buf, bs); if (!buf) return -1; } memcpy(buf + len, data, l); From 2f095515735b136d04d7f05713505e97a3d25aa6 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 1 Jul 2009 20:55:24 +0900 Subject: [PATCH 0230/1172] Fix memory leak. Remove stream packing feature. Add errorcheck in packing. --- msgpack/_msgpack.pyx | 135 +++++++++++++++++++++++-------------------- msgpack/pack.h | 2 +- 2 files changed, 73 insertions(+), 64 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 9d766c5..c455394 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -5,20 +5,22 @@ import cStringIO cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject + cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) - PyObject* Py_True - PyObject* Py_False - char* PyString_AsString(object o) - int PyMapping_Check(object o) - int PySequence_Check(object o) - long long PyLong_AsLongLong(object o) - unsigned long long PyLong_AsUnsignedLongLong(object o) - int PyLong_Check(object o) - int PyInt_Check(object o) - int PyFloat_Check(object o) - int PyString_Check(object o) - int PyUnicode_Check(object o) - object PyBuffer_FromMemory(const_char_ptr b, Py_ssize_t len) + cdef PyObject* Py_True + cdef PyObject* Py_False + + cdef char* PyString_AsString(object o) + cdef long long PyLong_AsLongLong(object o) + cdef unsigned long long PyLong_AsUnsignedLongLong(object o) + + cdef int PyMapping_Check(object o) + cdef int PySequence_Check(object o) + cdef int PyLong_Check(object o) + cdef int PyInt_Check(object o) + cdef int PyFloat_Check(object o) + cdef int PyString_Check(object o) + cdef int PyUnicode_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -50,101 +52,101 @@ cdef extern from "pack.h": cdef class Packer(object): - """Packer that pack data into strm. + """MessagePack Packer + + usage: - strm must have `write(bytes)` method. - size specifies local buffer size. + packer = Packer() + astream.write(packer.pack(a)) + astream.write(packer.pack(b)) """ cdef msgpack_packer pk - cdef object strm - cdef object writer - - def __init__(self, strm): - self.strm = strm + def __cinit__(self): cdef int buf_size = 1024*1024 self.pk.buf = malloc(buf_size); self.pk.buf_size = buf_size self.pk.length = 0 - def __del__(self): + def __dealloc__(self): free(self.pk.buf); - cdef __pack(self, object o): + cdef int __pack(self, object o): cdef long long llval cdef unsigned long long ullval cdef long longval cdef double fval cdef char* rawval + cdef int ret if o is None: - msgpack_pack_nil(&self.pk) + ret = msgpack_pack_nil(&self.pk) elif o == Py_True: - msgpack_pack_true(&self.pk) + ret = msgpack_pack_true(&self.pk) elif o == Py_False: - msgpack_pack_false(&self.pk) + ret = msgpack_pack_false(&self.pk) elif PyLong_Check(o): if o > 0: ullval = PyLong_AsUnsignedLongLong(o) - msgpack_pack_unsigned_long_long(&self.pk, ullval) + ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) else: llval = PyLong_AsLongLong(o) - msgpack_pack_long_long(&self.pk, llval) + ret = msgpack_pack_long_long(&self.pk, llval) elif PyInt_Check(o): longval = o - msgpack_pack_long(&self.pk, longval) + ret = msgpack_pack_long(&self.pk, longval) elif PyFloat_Check(o): fval = o - msgpack_pack_double(&self.pk, fval) + ret = msgpack_pack_double(&self.pk, fval) elif PyString_Check(o): rawval = o - msgpack_pack_raw(&self.pk, len(o)) - msgpack_pack_raw_body(&self.pk, rawval, len(o)) + ret = msgpack_pack_raw(&self.pk, len(o)) + if ret == 0: + ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyUnicode_Check(o): o = o.encode('utf-8') rawval = o - msgpack_pack_raw(&self.pk, len(o)) - msgpack_pack_raw_body(&self.pk, rawval, len(o)) + ret = msgpack_pack_raw(&self.pk, len(o)) + if ret == 0: + ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyMapping_Check(o): - msgpack_pack_map(&self.pk, len(o)) - for k,v in o.iteritems(): - self.__pack(k) - self.__pack(v) + ret = msgpack_pack_map(&self.pk, len(o)) + if ret == 0: + for k,v in o.iteritems(): + ret = self.__pack(k) + if ret != 0: break + ret = self.__pack(v) + if ret != 0: break elif PySequence_Check(o): - msgpack_pack_array(&self.pk, len(o)) - for v in o: - self.__pack(v) + ret = msgpack_pack_array(&self.pk, len(o)) + if ret == 0: + for v in o: + ret = self.__pack(v) + if ret != 0: break else: # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) + return ret - def pack(self, object obj, object flush=True): - self.__pack(obj) - buf = PyBuffer_FromMemory(self.pk.buf, self.pk.length) + def pack(self, object obj): + cdef int ret + ret = self.__pack(obj) + if ret: + raise TypeError + buf = PyString_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 - self.strm.write(buf) - if flush: - self.flush() + return buf - def flush(self): - """Flash local buffer and output stream if it has 'flush()' method.""" - if hasattr(self.strm, 'flush'): - self.strm.flush() - - close = flush def pack(object o, object stream): - u"""pack o and write to stream).""" - packer = Packer(stream) - packer.pack(o) - packer.flush() + """pack a object `o` and write it to stream).""" + packer = Packer() + stream.write(packer.pack(o)) def packb(object o): - u"""pack o and return packed bytes.""" - buf = cStringIO.StringIO() - packer = Packer(buf) - packer.pack(o) - return buf.getvalue() + """pack o and return packed bytes.""" + packer = Packer() + return packer.pack(o) packs = packb @@ -222,7 +224,14 @@ cdef class Unpacker(object): cdef int read_size cdef object waiting_bytes - def __init__(self, file_like=None, int read_size=4096): + def __cinit__(self): + self.buf = NULL + + def __dealloc__(self): + if self.buf: + free(self.buf); + + def __init__(self, file_like=None, int read_size=1024*1024): self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] diff --git a/msgpack/pack.h b/msgpack/pack.h index d7e0867..58f021e 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -72,7 +72,7 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ if (len + l > bs) { bs = (len + l) * 2; - buf = realloc(pk->buf, bs); + buf = realloc(buf, bs); if (!buf) return -1; } memcpy(buf + len, data, l); From 601209c83c11d9549eab86b708c6fe2f82d59f64 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 2 Jul 2009 14:25:48 +0900 Subject: [PATCH 0231/1172] support $Data::MessagePack::PreferredInteger for Data::Model --- perl/lib/Data/MessagePack.pm | 10 ++++++++++ perl/pack.c | 17 ++++++++++++++++ perl/t/05_preferred_int.t | 38 ++++++++++++++++++++++++++++++++++++ perl/t/data.pl | 5 ++++- 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 perl/t/05_preferred_int.t diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index cb3f73a..082e232 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -24,6 +24,16 @@ Data::MessagePack - messagepack Data::MessagePack is a binary packer for perl. +=head1 Configuration Variables + +=over 4 + +=item $Data::MessagePack::PreferredInteger + +Pack the string as int when the value looks like int(EXPERIMENTAL). + +=back + =head1 AUTHORS Tokuhiro Matsuno diff --git a/perl/pack.c b/perl/pack.c index 5aec963..aee3bd6 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -51,6 +51,16 @@ void need(enc_t *enc, STRLEN len) } } +static int looks_like_int(const char *str, size_t len) { + int i; + for (i=0; ipack($_[0])); + s/(..)/$1 /g; + s/ $//; + $_; +} + +sub pis ($$) { + is packit($_[0]), $_[1], 'dump ' . $_[1]; + # is(Dumper(Data::MessagePack->unpack(Data::MessagePack->pack($_[0]))), Dumper($_[0])); +} + +my @dat = ( + '0', '00', + '1', '01', + '10', '0a', + ''.0xEFFF => 'cd ef ff', + ''.0xFFFF => 'cd ff ff', + ''.0xFFFFFF => 'ce 00 ff ff ff', + ''.0xFFFFFFFF => 'aa 34 32 39 34 39 36 37 32 39 35', + ''.0xFFFFFFFFF => 'ab 36 38 37 31 39 34 37 36 37 33 35', + ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => 'b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34', + {'0' => '1'}, '81 00 01', + {'abc' => '1'}, '81 a3 61 62 63 01', +); +plan tests => 1*(scalar(@dat)/2); + +$Data::MessagePack::PreferredInteger = 1; +for (my $i=0; $i [undef, false, true], '94 a0 a1 61 a2 62 63 a3 64 65 66', ["", "a", "bc", "def"], @@ -12,4 +12,7 @@ no warnings 'uninitialized'; # i need this. i need this. '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff', [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], '82 c2 81 c0 c0 c3 81 c0 80', {false,{undef,undef}, true,{undef,{}}}, '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2', [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}], + 'ce 00 ff ff ff' => ''.0xFFFFFF, + 'aa 34 32 39 34 39 36 37 32 39 35' => ''.0xFFFFFFFF, + 'ab 36 38 37 31 39 34 37 36 37 33 35' => ''.0xFFFFFFFFF, ) From 7b198d5966ee8f486425dd625fae485bc250975d Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 2 Jul 2009 14:29:49 +0900 Subject: [PATCH 0232/1172] s/Preferred/Prefer/g suggested by kazuho++ --- perl/lib/Data/MessagePack.pm | 2 +- perl/pack.c | 2 +- perl/t/05_preferred_int.t | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 082e232..e7077f5 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -28,7 +28,7 @@ Data::MessagePack is a binary packer for perl. =over 4 -=item $Data::MessagePack::PreferredInteger +=item $Data::MessagePack::PreferInteger Pack the string as int when the value looks like int(EXPERIMENTAL). diff --git a/perl/pack.c b/perl/pack.c index aee3bd6..c2d871a 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -132,7 +132,7 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { STRLEN len; char * cval = SvPV(val, len); - SV* pref_int = get_sv("Data::MessagePack::PreferredInteger", 0); + SV* pref_int = get_sv("Data::MessagePack::PreferInteger", 0); if (pref_int && SvTRUE(pref_int) && looks_like_int(cval, len) && SvUV(val) < U32_MAX) { PACK_WRAPPER(uint32)(enc, SvUV(val)); return; diff --git a/perl/t/05_preferred_int.t b/perl/t/05_preferred_int.t index 39167c1..176430e 100644 --- a/perl/t/05_preferred_int.t +++ b/perl/t/05_preferred_int.t @@ -31,7 +31,7 @@ my @dat = ( ); plan tests => 1*(scalar(@dat)/2); -$Data::MessagePack::PreferredInteger = 1; +$Data::MessagePack::PreferInteger = 1; for (my $i=0; $i Date: Thu, 2 Jul 2009 14:37:54 +0900 Subject: [PATCH 0233/1172] Checking in changes prior to tagging of version 0.02. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 18041d4..fb31a69 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.02 + + - added $Data::MessagePack::PreferInteger + (requested by yappo++) + 0.01 - initial release to CPAN --- perl/Changes | 5 +++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 18041d4..fb31a69 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.02 + + - added $Data::MessagePack::PreferInteger + (requested by yappo++) + 0.01 - initial release to CPAN diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index e7077f5..712d3b5 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.01'; +our $VERSION = '0.02'; XSLoader::load(__PACKAGE__, $VERSION); From e92e99c4d11cf35652c4184feeab5fa6ac29cb97 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 2 Jul 2009 14:43:22 +0900 Subject: [PATCH 0234/1172] Perl: I don't need CCFLAGS --- perl/Makefile.PL | 3 --- 1 file changed, 3 deletions(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index 167bebe..773d65d 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -6,12 +6,9 @@ perl_version '5.008005'; license 'perl'; can_cc or die "This module requires a C compiler"; -my $ccflags = '.'; - makemaker_args( OBJECT => '$(O_FILES)', LIBS => [''], - CCFLAGS => $ccflags, clean => { FILES => q{ *.stackdump From 580fbe77e9301786234ae47bfa0300736d9d8a84 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 2 Jul 2009 16:40:41 +0900 Subject: [PATCH 0235/1172] do not check the string longer than 10. --- perl/pack.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perl/pack.c b/perl/pack.c index c2d871a..c8987fe 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -131,9 +131,10 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { if (SvPOKp(val)) { STRLEN len; char * cval = SvPV(val, len); + const int U32_STRLEN = 10; /* length(0xFFFFFFFF) */ SV* pref_int = get_sv("Data::MessagePack::PreferInteger", 0); - if (pref_int && SvTRUE(pref_int) && looks_like_int(cval, len) && SvUV(val) < U32_MAX) { + if (pref_int && SvTRUE(pref_int) && len <= U32_STRLEN && looks_like_int(cval, len) && SvUV(val) < U32_MAX) { PACK_WRAPPER(uint32)(enc, SvUV(val)); return; } From ffef0a0b6fd6963e0465652c38185ead937e9545 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 2 Jul 2009 17:42:28 +0900 Subject: [PATCH 0236/1172] oops. fixed memory leaks in stream unpacker. --- perl/MessagePack.c | 3 ++- perl/unpack.c | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/perl/MessagePack.c b/perl/MessagePack.c index 3b33fee..e583526 100644 --- a/perl/MessagePack.c +++ b/perl/MessagePack.c @@ -18,6 +18,7 @@ XS(xs_unpacker_execute_limit); XS(xs_unpacker_is_finished); XS(xs_unpacker_data); XS(xs_unpacker_reset); +XS(xs_unpacker_destroy); XS(boot_Data__MessagePack) { dXSARGS; @@ -35,6 +36,6 @@ XS(boot_Data__MessagePack) { newXS("Data::MessagePack::Unpacker::is_finished", xs_unpacker_is_finished, __FILE__); newXS("Data::MessagePack::Unpacker::data", xs_unpacker_data, __FILE__); newXS("Data::MessagePack::Unpacker::reset", xs_unpacker_reset, __FILE__); - + newXS("Data::MessagePack::Unpacker::DESTROY", xs_unpacker_destroy, __FILE__); } diff --git a/perl/unpack.c b/perl/unpack.c index c2e32dd..5a28fef 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -288,3 +288,14 @@ XS(xs_unpacker_reset) { XSRETURN(0); } +XS(xs_unpacker_destroy) { + dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: $unpacker->DESTROY()"); + } + + UNPACKER(ST(0), mp); + Safefree(mp); + + XSRETURN(0); +} From 3446c475d753b54cb2103753610fbc6a9596cae4 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 2 Jul 2009 17:43:58 +0900 Subject: [PATCH 0237/1172] Checking in changes prior to tagging of version 0.03. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index fb31a69..0c170a3 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.03 + + - performance tuning for too long string + - fixed memory leaks in stream unpacker + 0.02 - added $Data::MessagePack::PreferInteger --- perl/Changes | 5 +++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index fb31a69..0c170a3 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.03 + + - performance tuning for too long string + - fixed memory leaks in stream unpacker + 0.02 - added $Data::MessagePack::PreferInteger diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 712d3b5..2368a5b 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.02'; +our $VERSION = '0.03'; XSLoader::load(__PACKAGE__, $VERSION); From 0f9dcb86100e34c7f10a19399cfa5dca3d4dcdbc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 3 Jul 2009 01:49:37 +0900 Subject: [PATCH 0238/1172] perl: make PreferInteger variable magical and remove get_sv from _msgpack_pack_sv --- perl/MessagePack.c | 4 ++++ perl/lib/Data/MessagePack.pm | 1 + perl/pack.c | 35 +++++++++++++++++++++++++++++++++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/perl/MessagePack.c b/perl/MessagePack.c index 3b33fee..f56d147 100644 --- a/perl/MessagePack.c +++ b/perl/MessagePack.c @@ -19,10 +19,14 @@ XS(xs_unpacker_is_finished); XS(xs_unpacker_data); XS(xs_unpacker_reset); +void boot_Data__MessagePack_pack(void); + XS(boot_Data__MessagePack) { dXSARGS; HV * stash; + boot_Data__MessagePack_pack(); + newXS("Data::MessagePack::pack", xs_pack, __FILE__); newXS("Data::MessagePack::unpack", xs_unpack, __FILE__); stash = gv_stashpvn("Data::MessagePack", strlen("Data::MessagePack"), TRUE); diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 712d3b5..ff179e8 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -5,6 +5,7 @@ use XSLoader; use 5.008001; our $VERSION = '0.02'; +our $PreferInteger = 0; XSLoader::load(__PACKAGE__, $VERSION); diff --git a/perl/pack.c b/perl/pack.c index c8987fe..03e6530 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -51,6 +51,38 @@ void need(enc_t *enc, STRLEN len) } } + +static int s_pref_int = 0; + +static int pref_int_set(pTHX_ SV* sv, MAGIC* mg) { + if (SvTRUE(sv)) { + s_pref_int = 1; + } else { + s_pref_int = 0; + } + return 0; +} + +MGVTBL pref_int_vtbl = { + NULL, + pref_int_set, + NULL, + NULL, + NULL, + NULL, + NULL, +#ifdef MGf_LOCAL + NULL, +#endif +}; + +void boot_Data__MessagePack_pack(void) { + SV* var = get_sv("Data::MessagePack::PreferInteger", 0); + sv_magicext(var, NULL, PERL_MAGIC_ext, &pref_int_vtbl, NULL, 0); + SvSETMAGIC(var); +} + + static int looks_like_int(const char *str, size_t len) { int i; for (i=0; i Date: Fri, 3 Jul 2009 02:08:22 +0900 Subject: [PATCH 0239/1172] perl: PreferInteger: faster string to integer conversion; support negative value --- perl/pack.c | 64 ++++++++++++++++++++++++++++++++------- perl/t/05_preferred_int.t | 13 ++++++-- 2 files changed, 64 insertions(+), 13 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index 03e6530..b329551 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -26,7 +26,7 @@ typedef struct { char *end; /* SvEND (sv) */ SV *sv; /* result scalar */ } enc_t; -void need(enc_t *enc, STRLEN len); +static void need(enc_t *enc, STRLEN len); #define msgpack_pack_user enc_t* @@ -41,7 +41,7 @@ void need(enc_t *enc, STRLEN len); #define PACK_WRAPPER(t) _PACK_WRAPPER(t) #define INIT_SIZE 32 /* initial scalar size to be allocated */ -void need(enc_t *enc, STRLEN len) +static void need(enc_t *enc, STRLEN len) { if (enc->cur + len >= enc->end) { STRLEN cur = enc->cur - (char *)SvPVX (enc->sv); @@ -83,16 +83,60 @@ void boot_Data__MessagePack_pack(void) { } -static int looks_like_int(const char *str, size_t len) { - int i; - for (i=0; i 11) { return 0; } + negative = 1; + ++p; + } else { + /* length(0xFFFFFFFF) == 10 */ + if (len > 10) { return 0; } } + +#if '9'=='8'+1 && '8'=='7'+1 && '7'=='6'+1 && '6'=='5'+1 && '5'=='4'+1 \ + && '4'=='3'+1 && '3'=='2'+1 && '2'=='1'+1 && '1'=='0'+1 + do { + unsigned int c = ((int)*(p++)) - '0'; + if (c > 9) { return 0; } + num = num * 10 + c; + } while(p < pe); +#else + do { + switch (*(p++)) { + case '0': num = num * 10 + 0; break; + case '1': num = num * 10 + 1; break; + case '2': num = num * 10 + 2; break; + case '3': num = num * 10 + 3; break; + case '4': num = num * 10 + 4; break; + case '5': num = num * 10 + 5; break; + case '6': num = num * 10 + 6; break; + case '7': num = num * 10 + 7; break; + case '8': num = num * 10 + 8; break; + case '9': num = num * 10 + 9; break; + default: return 0; + } + } while(p < pe); +#endif + + if (negative) { + if (num > 0x80000000) { return 0; } + msgpack_pack_int32(enc, ((int32_t)num) * -1); + } else { + if (num > 0xFFFFFFFF) { return 0; } + msgpack_pack_uint32(enc, (uint32_t)num); + } + return 1; } + static void _msgpack_pack_sv(enc_t *enc, SV* val) { if (val==NULL) { msgpack_pack_nil(enc); @@ -163,10 +207,8 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { if (SvPOKp(val)) { STRLEN len; char * cval = SvPV(val, len); - const int U32_STRLEN = 10; /* length(0xFFFFFFFF) */ - if (s_pref_int && len <= U32_STRLEN && looks_like_int(cval, len) && SvUV(val) < U32_MAX) { - PACK_WRAPPER(uint32)(enc, SvUV(val)); + if (s_pref_int && try_int(enc, cval, len)) { return; } diff --git a/perl/t/05_preferred_int.t b/perl/t/05_preferred_int.t index 176430e..39c725a 100644 --- a/perl/t/05_preferred_int.t +++ b/perl/t/05_preferred_int.t @@ -17,15 +17,24 @@ sub pis ($$) { } my @dat = ( + '', 'a0', '0', '00', '1', '01', - '10', '0a', + '10', '0a', + '-1', 'ff', + '-10', 'f6', + '-', 'a1 2d', ''.0xEFFF => 'cd ef ff', ''.0xFFFF => 'cd ff ff', ''.0xFFFFFF => 'ce 00 ff ff ff', - ''.0xFFFFFFFF => 'aa 34 32 39 34 39 36 37 32 39 35', + ''.0xFFFFFFFF => 'ce ff ff ff ff', ''.0xFFFFFFFFF => 'ab 36 38 37 31 39 34 37 36 37 33 35', ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => 'b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34', + '-'.0x8000000 => 'd2 f8 00 00 00', + '-'.0x80000000 => 'd2 80 00 00 00', + '-'.0x800000000 => 'ac 2d 33 34 33 35 39 37 33 38 33 36 38', + '-'.0x8000000000 => 'ad 2d 35 34 39 37 35 35 38 31 33 38 38 38', + '-'.0x800000000000000000000000000000 => 'b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35', {'0' => '1'}, '81 00 01', {'abc' => '1'}, '81 a3 61 62 63 01', ); From b45dc8db61a5f97369094592bcf80116c1019c37 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Fri, 3 Jul 2009 15:14:31 +0900 Subject: [PATCH 0240/1172] check SvROK first --- perl/pack.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index b329551..b1e5677 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -141,6 +141,9 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { if (val==NULL) { msgpack_pack_nil(enc); return; + } else if (SvROK(val)) { + _msgpack_pack_sv(enc, SvRV(val)); + return; } switch (SvTYPE(val)) { @@ -200,9 +203,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { } } break; - case SVt_RV: - _msgpack_pack_sv(enc, SvRV(val)); - break; default: if (SvPOKp(val)) { STRLEN len; From 6fb6283463db961677a00b1ea00552e5e9679d1c Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Fri, 3 Jul 2009 15:23:50 +0900 Subject: [PATCH 0241/1172] Checking in changes prior to tagging of version 0.04. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 0c170a3..33fb36c 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,11 @@ +0.04 + + - check SvROK first(reported by yappo++) + - PreferInteger: faster string to integer conversion; support negative value + (frsyuki++) + - make PreferInteger variable magical and remove get_sv from _msgpack_pack_sv + (frsyuki++) + 0.03 - performance tuning for too long string --- perl/Changes | 8 ++++++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 0c170a3..33fb36c 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,11 @@ +0.04 + + - check SvROK first(reported by yappo++) + - PreferInteger: faster string to integer conversion; support negative value + (frsyuki++) + - make PreferInteger variable magical and remove get_sv from _msgpack_pack_sv + (frsyuki++) + 0.03 - performance tuning for too long string diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index e204669..c61cac0 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.03'; +our $VERSION = '0.04'; our $PreferInteger = 0; XSLoader::load(__PACKAGE__, $VERSION); From ad7cf12128388b8432ffe33c2010d0d0580297c3 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Fri, 3 Jul 2009 17:27:44 +0900 Subject: [PATCH 0242/1172] perl: added test case for "the flag is working?" --- perl/t/05_preferred_int.t | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/perl/t/05_preferred_int.t b/perl/t/05_preferred_int.t index 39c725a..9860711 100644 --- a/perl/t/05_preferred_int.t +++ b/perl/t/05_preferred_int.t @@ -38,10 +38,20 @@ my @dat = ( {'0' => '1'}, '81 00 01', {'abc' => '1'}, '81 a3 61 62 63 01', ); -plan tests => 1*(scalar(@dat)/2); +plan tests => 1*(scalar(@dat)/2) + 2; -$Data::MessagePack::PreferInteger = 1; for (my $i=0; $i Date: Fri, 3 Jul 2009 18:09:38 +0900 Subject: [PATCH 0243/1172] Perl: check IV by SvIOK --- perl/pack.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index b1e5677..90b18aa 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -150,13 +150,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { case SVt_NULL: msgpack_pack_nil(enc); break; - case SVt_IV: - if (SvIOK_UV(val)) { - msgpack_pack_uint32(enc, SvUV(val)); - } else { - PACK_WRAPPER(IVTYPE)(enc, SvIV(val)); - } - break; case SVt_PVNV: { STRLEN len = 0; @@ -215,6 +208,10 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { msgpack_pack_raw(enc, len); msgpack_pack_raw_body(enc, cval, len); return; + } else if (SvIOK_UV(val)) { + msgpack_pack_uint32(enc, SvUV(val)); + } else if (SvIOK(val)) { + PACK_WRAPPER(IVTYPE)(enc, SvIV(val)); } else { sv_dump(val); Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); From cb5c878b77c9286cc6ca47dd04978550479ae7de Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Fri, 3 Jul 2009 18:15:45 +0900 Subject: [PATCH 0244/1172] Checking in changes prior to tagging of version 0.05. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 33fb36c..15603ea 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.05 + + - change type detection for old perl + 0.04 - check SvROK first(reported by yappo++) --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 33fb36c..15603ea 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.05 + + - change type detection for old perl + 0.04 - check SvROK first(reported by yappo++) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index c61cac0..46fb45d 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.04'; +our $VERSION = '0.05'; our $PreferInteger = 0; XSLoader::load(__PACKAGE__, $VERSION); From 661f27348188d15e841467ececd75e27cd77980b Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Fri, 3 Jul 2009 18:18:40 +0900 Subject: [PATCH 0245/1172] Perl: use SvNOK. --- perl/pack.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index 90b18aa..ff0ef2d 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -163,9 +163,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { } } break; - case SVt_NV: - PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); - break; case SVt_PVAV: { AV* ary = (AV*)val; @@ -212,6 +209,8 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { msgpack_pack_uint32(enc, SvUV(val)); } else if (SvIOK(val)) { PACK_WRAPPER(IVTYPE)(enc, SvIV(val)); + } else if (SvNOK(val)) { + PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); } else { sv_dump(val); Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); From 7da1a9b3cfa5687f24cdfde4068677675bc4a076 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Fri, 3 Jul 2009 18:19:29 +0900 Subject: [PATCH 0246/1172] Checking in changes prior to tagging of version 0.06. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 15603ea..1a51ddf 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.06 + + - use SvNOK. + 0.05 - change type detection for old perl --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 15603ea..1a51ddf 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.06 + + - use SvNOK. + 0.05 - change type detection for old perl diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 46fb45d..be22203 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.05'; +our $VERSION = '0.06'; our $PreferInteger = 0; XSLoader::load(__PACKAGE__, $VERSION); From 979efbb9501e14d43e2def8a80992494690a9d39 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 9 Jul 2009 13:41:04 +0900 Subject: [PATCH 0247/1172] Support MinGW. --- python/msgpack/unpack_define.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/msgpack/unpack_define.h b/python/msgpack/unpack_define.h index 63668c2..d997569 100644 --- a/python/msgpack/unpack_define.h +++ b/python/msgpack/unpack_define.h @@ -45,8 +45,21 @@ extern "C" { #endif #endif +#ifdef __WIN32__ +static inline uint16_t msgpack_betoh16(uint16_t x) { + return ((x << 8) & 0xff00U) | + ((x >> 8) & 0x00ffU); +} +static inline uint32_t msgpack_betoh32(uint32_t x) { + return ((x << 24) & 0xff000000UL ) | + ((x << 8) & 0x00ff0000UL ) | + ((x >> 8) & 0x0000ff00UL ) | + ((x >> 24) & 0x000000ffUL ); +} +#else #define msgpack_betoh16(x) ntohs(x) #define msgpack_betoh32(x) ntohl(x) +#endif #ifdef __LITTLE_ENDIAN__ #if defined(__bswap_64) From 77ca35a865bcc8cb9d7ed2fc8508cfbfb5b91dc6 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 9 Jul 2009 13:41:04 +0900 Subject: [PATCH 0248/1172] Support MinGW. --- msgpack/unpack_define.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 63668c2..d997569 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -45,8 +45,21 @@ extern "C" { #endif #endif +#ifdef __WIN32__ +static inline uint16_t msgpack_betoh16(uint16_t x) { + return ((x << 8) & 0xff00U) | + ((x >> 8) & 0x00ffU); +} +static inline uint32_t msgpack_betoh32(uint32_t x) { + return ((x << 24) & 0xff000000UL ) | + ((x << 8) & 0x00ff0000UL ) | + ((x >> 8) & 0x0000ff00UL ) | + ((x >> 24) & 0x000000ffUL ); +} +#else #define msgpack_betoh16(x) ntohs(x) #define msgpack_betoh32(x) ntohl(x) +#endif #ifdef __LITTLE_ENDIAN__ #if defined(__bswap_64) From 900785e1aa2e031a4496f8c2a30bb95d0c950b9b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 12 Jul 2009 09:29:11 +0900 Subject: [PATCH 0249/1172] Clean up --- python/README | 6 +++--- python/msgpack/_msgpack.pyx | 6 ++++-- python/setup.py | 17 +++++++++-------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/python/README b/python/README index 243def2..6b98982 100644 --- a/python/README +++ b/python/README @@ -13,11 +13,11 @@ You can read document in docstring after `import msgpack` INSTALL --------- -Cython_ is required to build the binding. +Cython_ is required to build msgpack. .. _Cython: http://www.cython.org/ -Posix +posix '''''' You can install msgpack in common way. @@ -28,7 +28,7 @@ Windows MessagePack requires gcc currently. So you need to prepare MinGW GCC. - $ python setup.py install -c mingw + $ python setup.py install -c mingw32 TEST ---- diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index c455394..b407b92 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -163,7 +163,7 @@ cdef extern from "unpack.h": object template_data(template_context* ctx) -def unpacks(object packed_bytes): +def unpackb(object packed_bytes): """Unpack packed_bytes to object. Returns unpacked object.""" cdef const_char_ptr p = packed_bytes cdef template_context ctx @@ -176,10 +176,12 @@ def unpacks(object packed_bytes): else: return None +unpacks = unpackb + def unpack(object stream): """unpack from stream.""" packed = stream.read() - return unpacks(packed) + return unpackb(packed) cdef class UnpackIterator(object): cdef object unpacker diff --git a/python/setup.py b/python/setup.py index faadb33..fa8cb3a 100755 --- a/python/setup.py +++ b/python/setup.py @@ -7,18 +7,14 @@ import os version = '0.0.1dev' -PACKAGE_ROOT = os.getcwdu() -INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') msgpack_mod = Extension('msgpack._msgpack', - sources=['msgpack/_msgpack.pyx'], - include_dirs=[INCLUDE_PATH]) + sources=['msgpack/_msgpack.pyx'] + ) -desc = 'MessagePack serializer/desirializer.' +desc = 'MessagePack (de)serializer.' long_desc = desc + """ -Python binding of MessagePack_. - -This package is under development. +MessagePack_ (de)serializer for Python. .. _MessagePack: http://msgpack.sourceforge.jp/ @@ -37,4 +33,9 @@ setup(name='msgpack', packages=['msgpack'], description=desc, long_description=long_desc, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + ] ) From e963ad55d27ae0f39bec605f3a0f0362061c9655 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 12 Jul 2009 09:29:11 +0900 Subject: [PATCH 0250/1172] Clean up --- README | 6 +++--- msgpack/_msgpack.pyx | 6 ++++-- setup.py | 17 +++++++++-------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/README b/README index 243def2..6b98982 100644 --- a/README +++ b/README @@ -13,11 +13,11 @@ You can read document in docstring after `import msgpack` INSTALL --------- -Cython_ is required to build the binding. +Cython_ is required to build msgpack. .. _Cython: http://www.cython.org/ -Posix +posix '''''' You can install msgpack in common way. @@ -28,7 +28,7 @@ Windows MessagePack requires gcc currently. So you need to prepare MinGW GCC. - $ python setup.py install -c mingw + $ python setup.py install -c mingw32 TEST ---- diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index c455394..b407b92 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -163,7 +163,7 @@ cdef extern from "unpack.h": object template_data(template_context* ctx) -def unpacks(object packed_bytes): +def unpackb(object packed_bytes): """Unpack packed_bytes to object. Returns unpacked object.""" cdef const_char_ptr p = packed_bytes cdef template_context ctx @@ -176,10 +176,12 @@ def unpacks(object packed_bytes): else: return None +unpacks = unpackb + def unpack(object stream): """unpack from stream.""" packed = stream.read() - return unpacks(packed) + return unpackb(packed) cdef class UnpackIterator(object): cdef object unpacker diff --git a/setup.py b/setup.py index faadb33..fa8cb3a 100755 --- a/setup.py +++ b/setup.py @@ -7,18 +7,14 @@ import os version = '0.0.1dev' -PACKAGE_ROOT = os.getcwdu() -INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') msgpack_mod = Extension('msgpack._msgpack', - sources=['msgpack/_msgpack.pyx'], - include_dirs=[INCLUDE_PATH]) + sources=['msgpack/_msgpack.pyx'] + ) -desc = 'MessagePack serializer/desirializer.' +desc = 'MessagePack (de)serializer.' long_desc = desc + """ -Python binding of MessagePack_. - -This package is under development. +MessagePack_ (de)serializer for Python. .. _MessagePack: http://msgpack.sourceforge.jp/ @@ -37,4 +33,9 @@ setup(name='msgpack', packages=['msgpack'], description=desc, long_description=long_desc, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + ] ) From e5c49dae13b26f2155dffffbc4e9915badbcecfb Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 12 Jul 2009 20:02:21 +0900 Subject: [PATCH 0251/1172] Release 0.1.0 --- python/README | 4 ++-- python/msgpack/_msgpack.pyx | 12 ++++++------ python/setup.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/python/README b/python/README index 6b98982..2b2ebe9 100644 --- a/python/README +++ b/python/README @@ -3,8 +3,8 @@ MessagePack Python Binding =========================== :author: Naoki INADA -:version: 0.0.1 -:date: 2009-06-27 +:version: 0.1.0 +:date: 2009-07-12 HOW TO USE ----------- diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index b407b92..7990a18 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -139,7 +139,7 @@ cdef class Packer(object): def pack(object o, object stream): - """pack a object `o` and write it to stream).""" + """pack an object `o` and write it to stream).""" packer = Packer() stream.write(packer.pack(o)) @@ -164,7 +164,7 @@ cdef extern from "unpack.h": def unpackb(object packed_bytes): - """Unpack packed_bytes to object. Returns unpacked object.""" + """Unpack packed_bytes to object. Returns an unpacked object.""" cdef const_char_ptr p = packed_bytes cdef template_context ctx cdef size_t off = 0 @@ -179,7 +179,7 @@ def unpackb(object packed_bytes): unpacks = unpackb def unpack(object stream): - """unpack from stream.""" + """unpack an object from stream.""" packed = stream.read() return unpackb(packed) @@ -196,14 +196,14 @@ cdef class UnpackIterator(object): return self cdef class Unpacker(object): - """Unpacker(file_like=None, read_size=4096) + """Unpacker(file_like=None, read_size=1024*1024) Streaming unpacker. file_like must have read(n) method. read_size is used like file_like.read(read_size) - If file_like is None, you can feed() bytes. feed() is useful - for unpack from non-blocking stream. + If file_like is None, you can ``feed()`` bytes. ``feed()`` is + useful for unpacking from non-blocking stream. exsample 1: unpacker = Unpacker(afile) diff --git a/python/setup.py b/python/setup.py index fa8cb3a..255e0db 100755 --- a/python/setup.py +++ b/python/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.0.1dev' +version = '0.1.0' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 2d885ad22afa24c5fbf62b5cf0d5970f4a4a2ff7 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 12 Jul 2009 20:02:21 +0900 Subject: [PATCH 0252/1172] Release 0.1.0 --- README | 4 ++-- msgpack/_msgpack.pyx | 12 ++++++------ setup.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README b/README index 6b98982..2b2ebe9 100644 --- a/README +++ b/README @@ -3,8 +3,8 @@ MessagePack Python Binding =========================== :author: Naoki INADA -:version: 0.0.1 -:date: 2009-06-27 +:version: 0.1.0 +:date: 2009-07-12 HOW TO USE ----------- diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index b407b92..7990a18 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -139,7 +139,7 @@ cdef class Packer(object): def pack(object o, object stream): - """pack a object `o` and write it to stream).""" + """pack an object `o` and write it to stream).""" packer = Packer() stream.write(packer.pack(o)) @@ -164,7 +164,7 @@ cdef extern from "unpack.h": def unpackb(object packed_bytes): - """Unpack packed_bytes to object. Returns unpacked object.""" + """Unpack packed_bytes to object. Returns an unpacked object.""" cdef const_char_ptr p = packed_bytes cdef template_context ctx cdef size_t off = 0 @@ -179,7 +179,7 @@ def unpackb(object packed_bytes): unpacks = unpackb def unpack(object stream): - """unpack from stream.""" + """unpack an object from stream.""" packed = stream.read() return unpackb(packed) @@ -196,14 +196,14 @@ cdef class UnpackIterator(object): return self cdef class Unpacker(object): - """Unpacker(file_like=None, read_size=4096) + """Unpacker(file_like=None, read_size=1024*1024) Streaming unpacker. file_like must have read(n) method. read_size is used like file_like.read(read_size) - If file_like is None, you can feed() bytes. feed() is useful - for unpack from non-blocking stream. + If file_like is None, you can ``feed()`` bytes. ``feed()`` is + useful for unpacking from non-blocking stream. exsample 1: unpacker = Unpacker(afile) diff --git a/setup.py b/setup.py index fa8cb3a..255e0db 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.0.1dev' +version = '0.1.0' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 294e3fe7ab01ef9273b364b0d3d9df4e9b275158 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 13 Jul 2009 14:27:57 +0900 Subject: [PATCH 0253/1172] Add setup script for distribution. --- python/setup.py | 7 ++++--- python/setup_dev.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) create mode 100755 python/setup_dev.py diff --git a/python/setup.py b/python/setup.py index 255e0db..c0d121b 100755 --- a/python/setup.py +++ b/python/setup.py @@ -2,13 +2,14 @@ # coding: utf-8 from distutils.core import setup, Extension -from Cython.Distutils import build_ext +#from Cython.Distutils import build_ext import os version = '0.1.0' msgpack_mod = Extension('msgpack._msgpack', - sources=['msgpack/_msgpack.pyx'] + #sources=['msgpack/_msgpack.pyx'] + sources=['msgpack/_msgpack.c'] ) desc = 'MessagePack (de)serializer.' @@ -28,7 +29,7 @@ setup(name='msgpack', author='Naoki INADA', author_email='songofacandy@gmail.com', version=version, - cmdclass={'build_ext': build_ext}, + #cmdclass={'build_ext': build_ext}, ext_modules=[msgpack_mod], packages=['msgpack'], description=desc, diff --git a/python/setup_dev.py b/python/setup_dev.py new file mode 100755 index 0000000..255e0db --- /dev/null +++ b/python/setup_dev.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# coding: utf-8 + +from distutils.core import setup, Extension +from Cython.Distutils import build_ext +import os + +version = '0.1.0' + +msgpack_mod = Extension('msgpack._msgpack', + sources=['msgpack/_msgpack.pyx'] + ) + +desc = 'MessagePack (de)serializer.' +long_desc = desc + """ + +MessagePack_ (de)serializer for Python. + +.. _MessagePack: http://msgpack.sourceforge.jp/ + +What's MessagePack? (from http://msgpack.sourceforge.jp/) + + MessagePack is a binary-based efficient data interchange format that is + focused on high performance. It is like JSON, but very fast and small. +""" + +setup(name='msgpack', + author='Naoki INADA', + author_email='songofacandy@gmail.com', + version=version, + cmdclass={'build_ext': build_ext}, + ext_modules=[msgpack_mod], + packages=['msgpack'], + description=desc, + long_description=long_desc, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + ] + ) From 7247238873d75e30b5d4d823bb39869326e89f6a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 13 Jul 2009 14:27:57 +0900 Subject: [PATCH 0254/1172] Add setup script for distribution. --- setup.py | 7 ++++--- setup_dev.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) create mode 100755 setup_dev.py diff --git a/setup.py b/setup.py index 255e0db..c0d121b 100755 --- a/setup.py +++ b/setup.py @@ -2,13 +2,14 @@ # coding: utf-8 from distutils.core import setup, Extension -from Cython.Distutils import build_ext +#from Cython.Distutils import build_ext import os version = '0.1.0' msgpack_mod = Extension('msgpack._msgpack', - sources=['msgpack/_msgpack.pyx'] + #sources=['msgpack/_msgpack.pyx'] + sources=['msgpack/_msgpack.c'] ) desc = 'MessagePack (de)serializer.' @@ -28,7 +29,7 @@ setup(name='msgpack', author='Naoki INADA', author_email='songofacandy@gmail.com', version=version, - cmdclass={'build_ext': build_ext}, + #cmdclass={'build_ext': build_ext}, ext_modules=[msgpack_mod], packages=['msgpack'], description=desc, diff --git a/setup_dev.py b/setup_dev.py new file mode 100755 index 0000000..255e0db --- /dev/null +++ b/setup_dev.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# coding: utf-8 + +from distutils.core import setup, Extension +from Cython.Distutils import build_ext +import os + +version = '0.1.0' + +msgpack_mod = Extension('msgpack._msgpack', + sources=['msgpack/_msgpack.pyx'] + ) + +desc = 'MessagePack (de)serializer.' +long_desc = desc + """ + +MessagePack_ (de)serializer for Python. + +.. _MessagePack: http://msgpack.sourceforge.jp/ + +What's MessagePack? (from http://msgpack.sourceforge.jp/) + + MessagePack is a binary-based efficient data interchange format that is + focused on high performance. It is like JSON, but very fast and small. +""" + +setup(name='msgpack', + author='Naoki INADA', + author_email='songofacandy@gmail.com', + version=version, + cmdclass={'build_ext': build_ext}, + ext_modules=[msgpack_mod], + packages=['msgpack'], + description=desc, + long_description=long_desc, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + ] + ) From 6083bad5ffd646f715d756e7189041d542ee8ab0 Mon Sep 17 00:00:00 2001 From: inada-n Date: Mon, 13 Jul 2009 14:46:53 +0900 Subject: [PATCH 0255/1172] Fix wrong MANIFEST. --- python/MANIFEST | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/MANIFEST b/python/MANIFEST index 1fe22ec..f6c43b0 100644 --- a/python/MANIFEST +++ b/python/MANIFEST @@ -3,7 +3,7 @@ msgpack/pack.h msgpack/unpack.h msgpack/_msgpack.pyx msgpack/__init__.py -include/msgpack/pack_define.h -include/msgpack/pack_template.h -include/msgpack/unpack_define.h -include/msgpack/unpack_template.h +msgpack/pack_define.h +msgpack/pack_template.h +msgpack/unpack_define.h +msgpack/unpack_template.h From 6e0a7bdb293e53d5c1db226759ba77caec337715 Mon Sep 17 00:00:00 2001 From: inada-n Date: Mon, 13 Jul 2009 14:46:53 +0900 Subject: [PATCH 0256/1172] Fix wrong MANIFEST. --- MANIFEST | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/MANIFEST b/MANIFEST index 1fe22ec..f6c43b0 100644 --- a/MANIFEST +++ b/MANIFEST @@ -3,7 +3,7 @@ msgpack/pack.h msgpack/unpack.h msgpack/_msgpack.pyx msgpack/__init__.py -include/msgpack/pack_define.h -include/msgpack/pack_template.h -include/msgpack/unpack_define.h -include/msgpack/unpack_template.h +msgpack/pack_define.h +msgpack/pack_template.h +msgpack/unpack_define.h +msgpack/unpack_template.h From 0db5e0439dab0445d9f84aa5f6fbf2f535cd8d0d Mon Sep 17 00:00:00 2001 From: inada-n Date: Mon, 13 Jul 2009 15:49:38 +0900 Subject: [PATCH 0257/1172] Fix: Unpacker.unpack() may raise StopIteration before unpacking large object when deserializing from file. --- python/msgpack/_msgpack.pyx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 7990a18..cb95146 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -265,7 +265,11 @@ cdef class Unpacker(object): cdef Py_ssize_t add_size if self.file_like is not None: - self.waiting_bytes.append(self.file_like.read(self.read_size)) + next_bytes = self.file_like.read(self.read_size) + if next_bytes: + self.waiting_bytes.append(next_bytes) + else: + self.file_like = None if not self.waiting_bytes: return @@ -307,6 +311,8 @@ cdef class Unpacker(object): if ret == 1: return template_data(&self.ctx) elif ret == 0: + if self.file_like is not None: + return self.unpack() raise StopIteration, "No more unpack data." else: raise ValueError, "Unpack failed." From dc4e20ed35b6910211366e661382865a33fb2623 Mon Sep 17 00:00:00 2001 From: inada-n Date: Mon, 13 Jul 2009 15:49:38 +0900 Subject: [PATCH 0258/1172] Fix: Unpacker.unpack() may raise StopIteration before unpacking large object when deserializing from file. --- msgpack/_msgpack.pyx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 7990a18..cb95146 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -265,7 +265,11 @@ cdef class Unpacker(object): cdef Py_ssize_t add_size if self.file_like is not None: - self.waiting_bytes.append(self.file_like.read(self.read_size)) + next_bytes = self.file_like.read(self.read_size) + if next_bytes: + self.waiting_bytes.append(next_bytes) + else: + self.file_like = None if not self.waiting_bytes: return @@ -307,6 +311,8 @@ cdef class Unpacker(object): if ret == 1: return template_data(&self.ctx) elif ret == 0: + if self.file_like is not None: + return self.unpack() raise StopIteration, "No more unpack data." else: raise ValueError, "Unpack failed." From d19c4889293ef15f00ebe55c4246eaf8c48f44cc Mon Sep 17 00:00:00 2001 From: inada-n Date: Mon, 13 Jul 2009 15:52:25 +0900 Subject: [PATCH 0259/1172] version: python-0.1.1 --- python/setup.py | 2 +- python/setup_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index c0d121b..b26b9e8 100755 --- a/python/setup.py +++ b/python/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension #from Cython.Distutils import build_ext import os -version = '0.1.0' +version = '0.1.1' msgpack_mod = Extension('msgpack._msgpack', #sources=['msgpack/_msgpack.pyx'] diff --git a/python/setup_dev.py b/python/setup_dev.py index 255e0db..e28ef25 100755 --- a/python/setup_dev.py +++ b/python/setup_dev.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.1.0' +version = '0.1.1dev' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 3a0851b94949d730b9e69696ffc9dabc2acf6b6c Mon Sep 17 00:00:00 2001 From: inada-n Date: Mon, 13 Jul 2009 15:52:25 +0900 Subject: [PATCH 0260/1172] version: python-0.1.1 --- setup.py | 2 +- setup_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c0d121b..b26b9e8 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension #from Cython.Distutils import build_ext import os -version = '0.1.0' +version = '0.1.1' msgpack_mod = Extension('msgpack._msgpack', #sources=['msgpack/_msgpack.pyx'] diff --git a/setup_dev.py b/setup_dev.py index 255e0db..e28ef25 100755 --- a/setup_dev.py +++ b/setup_dev.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.1.0' +version = '0.1.1dev' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 390c5ad8a0424968dbcc4b69750d2c31f1872b4f Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 22 Jul 2009 13:41:31 +0900 Subject: [PATCH 0261/1172] added test case. --- perl/t/01_pack.t | 2 ++ 1 file changed, 2 insertions(+) diff --git a/perl/t/01_pack.t b/perl/t/01_pack.t index 60c67f8..dbf7a9a 100644 --- a/perl/t/01_pack.t +++ b/perl/t/01_pack.t @@ -15,6 +15,8 @@ sub pis ($$) { my @dat = ( 0, '00', + (my $foo="0")+0, '00', + {2 => undef}, '81 a1 32 c0', 1, '01', 127, '7f', 128, 'cc 80', From 718227bc2cc7ed2d4a4152a1018f0fa046bbc57e Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 22 Jul 2009 16:40:15 +0900 Subject: [PATCH 0262/1172] Perl: do not use switch (SvTYPE) --- perl/pack.c | 114 ++++++++++++++++++++--------------------------- perl/t/01_pack.t | 1 + 2 files changed, 50 insertions(+), 65 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index ff0ef2d..50b82e5 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -140,81 +140,65 @@ static int try_int(enc_t* enc, const char *p, size_t len) { static void _msgpack_pack_sv(enc_t *enc, SV* val) { if (val==NULL) { msgpack_pack_nil(enc); - return; } else if (SvROK(val)) { _msgpack_pack_sv(enc, SvRV(val)); - return; - } - - switch (SvTYPE(val)) { - case SVt_NULL: - msgpack_pack_nil(enc); - break; - case SVt_PVNV: - { - STRLEN len = 0; - char *pv = SvPV(val, len); - if (len == 1 && *pv == '1') { - msgpack_pack_true(enc); - } else if (len == 0 && *pv==0) { - msgpack_pack_false(enc); + } else if (SVt_PVNV == SvTYPE(val)) { + STRLEN len = 0; + char *pv = SvPV(val, len); + if (len == 1 && *pv == '1') { + msgpack_pack_true(enc); + } else if (len == 0 && *pv==0) { + msgpack_pack_false(enc); + } else { + msgpack_pack_nil(enc); + } + } else if (SvTYPE(val) == SVt_PVAV) { + AV* ary = (AV*)val; + int len = av_len(ary) + 1; + int i; + msgpack_pack_array(enc, len); + for (i=0; i undef}, '81 a1 32 c0', + do {no warnings; my $foo = 10; "$foo"; $foo = undef; $foo} => 'c0', # PVIV but !POK && !IOK 1, '01', 127, '7f', 128, 'cc 80', From 794adf9469074cb4ca93c5b71d0a5ee7572d85b9 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 22 Jul 2009 16:42:12 +0900 Subject: [PATCH 0263/1172] Checking in changes prior to tagging of version 0.07. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 1a51ddf..4e7f1d7 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.07 + + - do not use switch (SvTYPE(val)). + 0.06 - use SvNOK. --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 1a51ddf..4e7f1d7 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.07 + + - do not use switch (SvTYPE(val)). + 0.06 - use SvNOK. diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index be22203..f01a92a 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.06'; +our $VERSION = '0.07'; our $PreferInteger = 0; XSLoader::load(__PACKAGE__, $VERSION); From 0fe79a7752f1a2dec796281a9a6639d126948bd3 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 30 Jul 2009 13:19:49 +0900 Subject: [PATCH 0264/1172] Perl: added benchmark script for memroy-usage --- perl/benchmark/size.pl | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 perl/benchmark/size.pl diff --git a/perl/benchmark/size.pl b/perl/benchmark/size.pl new file mode 100644 index 0000000..85555ce --- /dev/null +++ b/perl/benchmark/size.pl @@ -0,0 +1,28 @@ +#!/usr/bin/perl +use strict; +use warnings; +use Data::MessagePack; +use Storable; +use Text::SimpleTable; + +my @entries = ( + 1, + 3.14, + {}, + [], + [('a')x10], + {('a')x10}, + +{1,+{1,+{}}}, + +[+[+[]]], +); + +my $table = Text::SimpleTable->new([10, 'storable'], [10, 'msgpack']); + +for my $e (@entries) { + $table->row( + length(Storable::nfreeze(ref $e ? $e : \$e)), + length(Data::MessagePack->pack($e)), + ); +} + +print $table->draw; From 3275eee28173e9c4d00641204ab966e7fa278dcd Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 30 Jul 2009 13:24:38 +0900 Subject: [PATCH 0265/1172] Perl: display original data --- perl/benchmark/size.pl | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/perl/benchmark/size.pl b/perl/benchmark/size.pl index 85555ce..928daca 100644 --- a/perl/benchmark/size.pl +++ b/perl/benchmark/size.pl @@ -6,20 +6,24 @@ use Storable; use Text::SimpleTable; my @entries = ( - 1, - 3.14, - {}, - [], - [('a')x10], - {('a')x10}, - +{1,+{1,+{}}}, - +[+[+[]]], + '1', + '3.14', + '{}', + '[]', + "[('a')x10]", + "{('a')x10}", + "+{1,+{1,+{}}}", + "+[+[+[]]]", ); -my $table = Text::SimpleTable->new([10, 'storable'], [10, 'msgpack']); +my $table = Text::SimpleTable->new([15, 'src'], [9, 'storable'], [7, 'msgpack']); + +for my $src (@entries) { + my $e = eval $src; + die $@ if $@; -for my $e (@entries) { $table->row( + $src, length(Storable::nfreeze(ref $e ? $e : \$e)), length(Data::MessagePack->pack($e)), ); From bb455c1f71e859122e55673d23e8f1311852a82c Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 30 Jul 2009 13:29:24 +0900 Subject: [PATCH 0266/1172] perl: display module version --- perl/benchmark/size.pl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/perl/benchmark/size.pl b/perl/benchmark/size.pl index 928daca..cf5c1ce 100644 --- a/perl/benchmark/size.pl +++ b/perl/benchmark/size.pl @@ -29,4 +29,9 @@ for my $src (@entries) { ); } +print "perl: $]\n"; +print "Storable: $Storable::VERSION\n"; +print "Data::MessagePack: $Data::MessagePack::VERSION\n"; +print "\n"; print $table->draw; + From 4db90d23ac95592ea3892513026de326926007f9 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 30 Jul 2009 16:22:00 +0900 Subject: [PATCH 0267/1172] - fixed SVt_PVNV issue. --- perl/Makefile.PL | 4 ++ perl/MessagePack.c | 2 - perl/lib/Data/MessagePack.pm | 5 ++ perl/pack.c | 120 ++++++++++++++++++++++------------- perl/t/01_pack.t | 1 + perl/unpack.c | 58 +++++++++++------ perl/util.h | 11 ++++ 7 files changed, 133 insertions(+), 68 deletions(-) create mode 100644 perl/util.h diff --git a/perl/Makefile.PL b/perl/Makefile.PL index 773d65d..d1bff06 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -23,6 +23,10 @@ makemaker_args( tests 't/*.t'; author_tests('xt'); +if ($ENV{DEBUG}) { + cc_append_to_ccflags '-g'; +} + # copy modules if ($Module::Install::AUTHOR && -d File::Spec->catfile('..', 'msgpack')) { mkdir 'msgpack' unless -d 'msgpack'; diff --git a/perl/MessagePack.c b/perl/MessagePack.c index 6b9010c..fd1b344 100644 --- a/perl/MessagePack.c +++ b/perl/MessagePack.c @@ -31,8 +31,6 @@ XS(boot_Data__MessagePack) { newXS("Data::MessagePack::pack", xs_pack, __FILE__); newXS("Data::MessagePack::unpack", xs_unpack, __FILE__); stash = gv_stashpvn("Data::MessagePack", strlen("Data::MessagePack"), TRUE); - newCONSTSUB(stash, "true", &PL_sv_yes); - newCONSTSUB(stash, "false", &PL_sv_no); newXS("Data::MessagePack::Unpacker::new", xs_unpacker_new, __FILE__); newXS("Data::MessagePack::Unpacker::execute", xs_unpacker_execute, __FILE__); diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index f01a92a..38431eb 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -7,6 +7,11 @@ use 5.008001; our $VERSION = '0.07'; our $PreferInteger = 0; +our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; +our $false = do { bless \(my $dummy = 0), "Data::MessagePack::Boolean" }; +sub true () { $true } +sub false () { $false } + XSLoader::load(__PACKAGE__, $VERSION); 1; diff --git a/perl/pack.c b/perl/pack.c index 50b82e5..01dd5b6 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -137,23 +137,71 @@ static int try_int(enc_t* enc, const char *p, size_t len) { } -static void _msgpack_pack_sv(enc_t *enc, SV* val) { - if (val==NULL) { +static void _msgpack_pack_rv(enc_t *enc, SV* sv); + +static void _msgpack_pack_sv(enc_t *enc, SV* sv) { + SvGETMAGIC(sv); + + if (sv==NULL) { msgpack_pack_nil(enc); - } else if (SvROK(val)) { - _msgpack_pack_sv(enc, SvRV(val)); - } else if (SVt_PVNV == SvTYPE(val)) { - STRLEN len = 0; - char *pv = SvPV(val, len); - if (len == 1 && *pv == '1') { - msgpack_pack_true(enc); - } else if (len == 0 && *pv==0) { - msgpack_pack_false(enc); + } else if (SvPOKp(sv)) { + STRLEN len; + char * csv = SvPV(sv, len); + + if (s_pref_int && try_int(enc, csv, len)) { + return; } else { - msgpack_pack_nil(enc); + msgpack_pack_raw(enc, len); + msgpack_pack_raw_body(enc, csv, len); } - } else if (SvTYPE(val) == SVt_PVAV) { - AV* ary = (AV*)val; + } else if (SvNOKp(sv)) { + PACK_WRAPPER(NVTYPE)(enc, SvNVX(sv)); + } else if (SvIOK_UV(sv)) { + msgpack_pack_uint32(enc, SvUV(sv)); + } else if (SvIOKp(sv)) { + PACK_WRAPPER(IVTYPE)(enc, SvIV(sv)); + } else if (SvROK(sv)) { + _msgpack_pack_rv(enc, SvRV(sv)); + } else if (!SvOK(sv)) { + msgpack_pack_nil(enc); + } else if (isGV(sv)) { + Perl_croak(aTHX_ "msgpack cannot pack the GV\n"); + } else { + sv_dump(sv); + Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(sv)); + } +} + +static void _msgpack_pack_rv(enc_t *enc, SV* sv) { + svtype svt; + SvGETMAGIC(sv); + svt = SvTYPE(sv); + + if (SvOBJECT (sv)) { + HV *stash = gv_stashpv ("Data::MessagePack::Boolean", 1); // TODO: cache? + if (SvSTASH (sv) == stash) { + if (SvIV(sv)) { + msgpack_pack_true(enc); + } else { + msgpack_pack_false(enc); + } + } else { + croak ("encountered object '%s', Data::MessagePack doesn't allow the object", + SvPV_nolen(sv_2mortal(newRV_inc(sv)))); + } + } else if (svt == SVt_PVHV) { + HV* hval = (HV*)sv; + int count = hv_iterinit(hval); + HE* he; + + msgpack_pack_map(enc, count); + + while (he = hv_iternext(hval)) { + _msgpack_pack_sv(enc, hv_iterkeysv(he)); + _msgpack_pack_sv(enc, HeVAL(he)); + } + } else if (svt == SVt_PVAV) { + AV* ary = (AV*)sv; int len = av_len(ary) + 1; int i; msgpack_pack_array(enc, len); @@ -165,40 +213,22 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { msgpack_pack_nil(enc); } } - } else if (SvTYPE(val) == SVt_PVHV) { - HV* hval = (HV*)val; - int count = hv_iterinit(hval); - HE* he; + } else if (svt < SVt_PVAV) { + STRLEN len = 0; + char *pv = svt ? SvPV (sv, len) : 0; - msgpack_pack_map(enc, count); - - while (he = hv_iternext(hval)) { - _msgpack_pack_sv(enc, hv_iterkeysv(he)); - _msgpack_pack_sv(enc, HeVAL(he)); + if (len == 1 && *pv == '1') + msgpack_pack_true(enc); + else if (len == 1 && *pv == '0') + msgpack_pack_false(enc); + else { + sv_dump(sv); + croak("cannot encode reference to scalar '%s' unless the scalar is 0 or 1", + SvPV_nolen (sv_2mortal (newRV_inc (sv)))); } - } else if (SvPOKp(val)) { - STRLEN len; - char * cval = SvPV(val, len); - - if (s_pref_int && try_int(enc, cval, len)) { - return; - } - - msgpack_pack_raw(enc, len); - msgpack_pack_raw_body(enc, cval, len); - } else if (SvIOK_UV(val)) { - msgpack_pack_uint32(enc, SvUV(val)); - } else if (SvIOK(val)) { - PACK_WRAPPER(IVTYPE)(enc, SvIV(val)); - } else if (SvNOK(val)) { - PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); - } else if (!SvOK(val)) { - msgpack_pack_nil(enc); - } else if (isGV(val)) { - Perl_croak(aTHX_ "msgpack cannot pack the GV\n"); } else { - sv_dump(val); - Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); + croak ("encountered %s, but msgpack can only represent references to arrays or hashes", + SvPV_nolen (sv_2mortal (newRV_inc (sv)))); } } diff --git a/perl/t/01_pack.t b/perl/t/01_pack.t index 22a8c14..50fd663 100644 --- a/perl/t/01_pack.t +++ b/perl/t/01_pack.t @@ -33,6 +33,7 @@ my @dat = ( -32768, 'd1 80 00', -32769, 'd2 ff ff 7f ff', 1.0, 'cb 3f f0 00 00 00 00 00 00', + do { my $x=3.0;my $y = "$x";$x }, 'a1 33', # PVNV "", 'a0', "a", 'a1 61', "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 'bf 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61', diff --git a/perl/unpack.c b/perl/unpack.c index 5a28fef..7a4f422 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -5,6 +5,7 @@ extern "C" { #include "EXTERN.h" #include "perl.h" #include "XSUB.h" +#include "util.h" #define NEED_newRV_noinc #define NEED_sv_2pv_flags #include "ppport.h" @@ -33,6 +34,21 @@ typedef struct { #define msgpack_unpack_user unpack_user +/* ---------------------------------------------------------------------- */ +/* utility functions */ + +static INLINE SV * +get_bool (const char *name) { + SV * sv = get_sv( name, 1 ); + + SvREADONLY_on(sv); + SvREADONLY_on( SvRV(sv) ); + + return sv; +} + +/* ---------------------------------------------------------------------- */ + struct template_context; typedef struct template_context msgpack_unpack_t; @@ -43,61 +59,61 @@ static SV* template_data(msgpack_unpack_t* u); static int template_execute(msgpack_unpack_t* u, const char* data, size_t len, size_t* off); -static inline SV* template_callback_root(unpack_user* u) +static INLINE SV* template_callback_root(unpack_user* u) { return &PL_sv_undef; } -static inline int template_callback_uint8(unpack_user* u, uint8_t d, SV** o) +static INLINE int template_callback_uint8(unpack_user* u, uint8_t d, SV** o) { *o = newSVuv(d); return 0; } -static inline int template_callback_uint16(unpack_user* u, uint16_t d, SV** o) +static INLINE int template_callback_uint16(unpack_user* u, uint16_t d, SV** o) { *o = newSVuv(d); return 0; } -static inline int template_callback_uint32(unpack_user* u, uint32_t d, SV** o) +static INLINE int template_callback_uint32(unpack_user* u, uint32_t d, SV** o) { *o = newSVuv(d); return 0; } -static inline int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) +static INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) { *o = newSVuv(d); return 0; } -static inline int template_callback_int8(unpack_user* u, int8_t d, SV** o) +static INLINE int template_callback_int8(unpack_user* u, int8_t d, SV** o) { *o = newSViv((long)d); return 0; } -static inline int template_callback_int16(unpack_user* u, int16_t d, SV** o) +static INLINE int template_callback_int16(unpack_user* u, int16_t d, SV** o) { *o = newSViv((long)d); return 0; } -static inline int template_callback_int32(unpack_user* u, int32_t d, SV** o) +static INLINE int template_callback_int32(unpack_user* u, int32_t d, SV** o) { *o = newSViv((long)d); return 0; } -static inline int template_callback_int64(unpack_user* u, int64_t d, SV** o) +static INLINE int template_callback_int64(unpack_user* u, int64_t d, SV** o) { *o = newSViv(d); return 0; } -static inline int template_callback_float(unpack_user* u, float d, SV** o) +static INLINE int template_callback_float(unpack_user* u, float d, SV** o) { *o = newSVnv(d); return 0; } -static inline int template_callback_double(unpack_user* u, double d, SV** o) +static INLINE int template_callback_double(unpack_user* u, double d, SV** o) { *o = newSVnv(d); return 0; } -static inline int template_callback_nil(unpack_user* u, SV** o) +static INLINE int template_callback_nil(unpack_user* u, SV** o) { *o = &PL_sv_undef; return 0; } -static inline int template_callback_true(unpack_user* u, SV** o) -{ *o = &PL_sv_yes; return 0; } +static INLINE int template_callback_true(unpack_user* u, SV** o) +{ *o = get_bool("Data::MessagePack::true") ; return 0; } -static inline int template_callback_false(unpack_user* u, SV** o) -{ *o = &PL_sv_no; return 0;} +static INLINE int template_callback_false(unpack_user* u, SV** o) +{ *o = get_bool("Data::MessagePack::false") ; return 0; } -static inline int template_callback_array(unpack_user* u, unsigned int n, SV** o) +static INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o) { AV* a = newAV(); *o = (SV*)newRV_noinc((SV*)a); av_extend(a, n); return 0; } -static inline int template_callback_array_item(unpack_user* u, SV** c, SV* o) +static INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o) { av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */ -static inline int template_callback_map(unpack_user* u, unsigned int n, SV** o) +static INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o) { HV * h = newHV(); *o = newRV_noinc((SV*)h); return 0; } -static inline int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v) +static INLINE int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v) { hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; } -static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o) +static INLINE int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o) { *o = (l == 0) ? newSVpv("", 0) : newSVpv(p, l); return 0; } #define UNPACKER(from, name) \ diff --git a/perl/util.h b/perl/util.h new file mode 100644 index 0000000..2b4ed07 --- /dev/null +++ b/perl/util.h @@ -0,0 +1,11 @@ +#ifndef __PERL_MSGPACK_UTIL_H__ +#define __PERL_MSGPACK_UTIL_H__ + +#if __GNUC__ >= 3 +# define INLINE inline +#else +# define INLINE +#endif + +#endif // __PERL_MSGPACK_UTIL_H__ + From d3906bd1aba9996b548f49c35383c7099c64f09d Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 30 Jul 2009 16:38:03 +0900 Subject: [PATCH 0268/1172] Perl: use M::I::XSUtil --- perl/.gitignore | 1 + perl/Makefile.PL | 23 +- perl/ppport.h | 7063 ---------------------------------------------- 3 files changed, 10 insertions(+), 7077 deletions(-) delete mode 100644 perl/ppport.h diff --git a/perl/.gitignore b/perl/.gitignore index a9096b6..f060908 100644 --- a/perl/.gitignore +++ b/perl/.gitignore @@ -10,3 +10,4 @@ pack.o pm_to_blib unpack.o MANIFEST +ppport.h diff --git a/perl/Makefile.PL b/perl/Makefile.PL index d1bff06..dc74d23 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -6,22 +6,17 @@ perl_version '5.008005'; license 'perl'; can_cc or die "This module requires a C compiler"; -makemaker_args( - OBJECT => '$(O_FILES)', - LIBS => [''], - clean => { - FILES => q{ - *.stackdump - *.gcov *.gcda *.gcno - *.out - nytprof - cover_db - }, - }, -); - tests 't/*.t'; author_tests('xt'); +use_ppport 3.19; + +clean_files qw{ + *.stackdump + *.gcov *.gcda *.gcno + *.out + nytprof + cover_db +}; if ($ENV{DEBUG}) { cc_append_to_ccflags '-g'; diff --git a/perl/ppport.h b/perl/ppport.h deleted file mode 100644 index ec2f1cc..0000000 --- a/perl/ppport.h +++ /dev/null @@ -1,7063 +0,0 @@ -#if 0 -<<'SKIP'; -#endif -/* ----------------------------------------------------------------------- - - ppport.h -- Perl/Pollution/Portability Version 3.19 - - Automatically created by Devel::PPPort running under perl 5.010000. - - Do NOT edit this file directly! -- Edit PPPort_pm.PL and the - includes in parts/inc/ instead. - - Use 'perldoc ppport.h' to view the documentation below. - ----------------------------------------------------------------------- - -SKIP - -=pod - -=head1 NAME - -ppport.h - Perl/Pollution/Portability version 3.19 - -=head1 SYNOPSIS - - perl ppport.h [options] [source files] - - Searches current directory for files if no [source files] are given - - --help show short help - - --version show version - - --patch=file write one patch file with changes - --copy=suffix write changed copies with suffix - --diff=program use diff program and options - - --compat-version=version provide compatibility with Perl version - --cplusplus accept C++ comments - - --quiet don't output anything except fatal errors - --nodiag don't show diagnostics - --nohints don't show hints - --nochanges don't suggest changes - --nofilter don't filter input files - - --strip strip all script and doc functionality from - ppport.h - - --list-provided list provided API - --list-unsupported list unsupported API - --api-info=name show Perl API portability information - -=head1 COMPATIBILITY - -This version of F is designed to support operation with Perl -installations back to 5.003, and has been tested up to 5.10.0. - -=head1 OPTIONS - -=head2 --help - -Display a brief usage summary. - -=head2 --version - -Display the version of F. - -=head2 --patch=I - -If this option is given, a single patch file will be created if -any changes are suggested. This requires a working diff program -to be installed on your system. - -=head2 --copy=I - -If this option is given, a copy of each file will be saved with -the given suffix that contains the suggested changes. This does -not require any external programs. Note that this does not -automagially add a dot between the original filename and the -suffix. If you want the dot, you have to include it in the option -argument. - -If neither C<--patch> or C<--copy> are given, the default is to -simply print the diffs for each file. This requires either -C or a C program to be installed. - -=head2 --diff=I - -Manually set the diff program and options to use. The default -is to use C, when installed, and output unified -context diffs. - -=head2 --compat-version=I - -Tell F to check for compatibility with the given -Perl version. The default is to check for compatibility with Perl -version 5.003. You can use this option to reduce the output -of F if you intend to be backward compatible only -down to a certain Perl version. - -=head2 --cplusplus - -Usually, F will detect C++ style comments and -replace them with C style comments for portability reasons. -Using this option instructs F to leave C++ -comments untouched. - -=head2 --quiet - -Be quiet. Don't print anything except fatal errors. - -=head2 --nodiag - -Don't output any diagnostic messages. Only portability -alerts will be printed. - -=head2 --nohints - -Don't output any hints. Hints often contain useful portability -notes. Warnings will still be displayed. - -=head2 --nochanges - -Don't suggest any changes. Only give diagnostic output and hints -unless these are also deactivated. - -=head2 --nofilter - -Don't filter the list of input files. By default, files not looking -like source code (i.e. not *.xs, *.c, *.cc, *.cpp or *.h) are skipped. - -=head2 --strip - -Strip all script and documentation functionality from F. -This reduces the size of F dramatically and may be useful -if you want to include F in smaller modules without -increasing their distribution size too much. - -The stripped F will have a C<--unstrip> option that allows -you to undo the stripping, but only if an appropriate C -module is installed. - -=head2 --list-provided - -Lists the API elements for which compatibility is provided by -F. Also lists if it must be explicitly requested, -if it has dependencies, and if there are hints or warnings for it. - -=head2 --list-unsupported - -Lists the API elements that are known not to be supported by -F and below which version of Perl they probably -won't be available or work. - -=head2 --api-info=I - -Show portability information for API elements matching I. -If I is surrounded by slashes, it is interpreted as a regular -expression. - -=head1 DESCRIPTION - -In order for a Perl extension (XS) module to be as portable as possible -across differing versions of Perl itself, certain steps need to be taken. - -=over 4 - -=item * - -Including this header is the first major one. This alone will give you -access to a large part of the Perl API that hasn't been available in -earlier Perl releases. Use - - perl ppport.h --list-provided - -to see which API elements are provided by ppport.h. - -=item * - -You should avoid using deprecated parts of the API. For example, using -global Perl variables without the C prefix is deprecated. Also, -some API functions used to have a C prefix. Using this form is -also deprecated. You can safely use the supported API, as F -will provide wrappers for older Perl versions. - -=item * - -If you use one of a few functions or variables that were not present in -earlier versions of Perl, and that can't be provided using a macro, you -have to explicitly request support for these functions by adding one or -more C<#define>s in your source code before the inclusion of F. - -These functions or variables will be marked C in the list shown -by C<--list-provided>. - -Depending on whether you module has a single or multiple files that -use such functions or variables, you want either C or global -variants. - -For a C function or variable (used only in a single source -file), use: - - #define NEED_function - #define NEED_variable - -For a global function or variable (used in multiple source files), -use: - - #define NEED_function_GLOBAL - #define NEED_variable_GLOBAL - -Note that you mustn't have more than one global request for the -same function or variable in your project. - - Function / Variable Static Request Global Request - ----------------------------------------------------------------------------------------- - PL_parser NEED_PL_parser NEED_PL_parser_GLOBAL - PL_signals NEED_PL_signals NEED_PL_signals_GLOBAL - eval_pv() NEED_eval_pv NEED_eval_pv_GLOBAL - grok_bin() NEED_grok_bin NEED_grok_bin_GLOBAL - grok_hex() NEED_grok_hex NEED_grok_hex_GLOBAL - grok_number() NEED_grok_number NEED_grok_number_GLOBAL - grok_numeric_radix() NEED_grok_numeric_radix NEED_grok_numeric_radix_GLOBAL - grok_oct() NEED_grok_oct NEED_grok_oct_GLOBAL - load_module() NEED_load_module NEED_load_module_GLOBAL - my_snprintf() NEED_my_snprintf NEED_my_snprintf_GLOBAL - my_sprintf() NEED_my_sprintf NEED_my_sprintf_GLOBAL - my_strlcat() NEED_my_strlcat NEED_my_strlcat_GLOBAL - my_strlcpy() NEED_my_strlcpy NEED_my_strlcpy_GLOBAL - newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL - newRV_noinc() NEED_newRV_noinc NEED_newRV_noinc_GLOBAL - newSV_type() NEED_newSV_type NEED_newSV_type_GLOBAL - newSVpvn_flags() NEED_newSVpvn_flags NEED_newSVpvn_flags_GLOBAL - newSVpvn_share() NEED_newSVpvn_share NEED_newSVpvn_share_GLOBAL - pv_display() NEED_pv_display NEED_pv_display_GLOBAL - pv_escape() NEED_pv_escape NEED_pv_escape_GLOBAL - pv_pretty() NEED_pv_pretty NEED_pv_pretty_GLOBAL - sv_2pv_flags() NEED_sv_2pv_flags NEED_sv_2pv_flags_GLOBAL - sv_2pvbyte() NEED_sv_2pvbyte NEED_sv_2pvbyte_GLOBAL - sv_catpvf_mg() NEED_sv_catpvf_mg NEED_sv_catpvf_mg_GLOBAL - sv_catpvf_mg_nocontext() NEED_sv_catpvf_mg_nocontext NEED_sv_catpvf_mg_nocontext_GLOBAL - sv_pvn_force_flags() NEED_sv_pvn_force_flags NEED_sv_pvn_force_flags_GLOBAL - sv_setpvf_mg() NEED_sv_setpvf_mg NEED_sv_setpvf_mg_GLOBAL - sv_setpvf_mg_nocontext() NEED_sv_setpvf_mg_nocontext NEED_sv_setpvf_mg_nocontext_GLOBAL - vload_module() NEED_vload_module NEED_vload_module_GLOBAL - vnewSVpvf() NEED_vnewSVpvf NEED_vnewSVpvf_GLOBAL - warner() NEED_warner NEED_warner_GLOBAL - -To avoid namespace conflicts, you can change the namespace of the -explicitly exported functions / variables using the C -macro. Just C<#define> the macro before including C: - - #define DPPP_NAMESPACE MyOwnNamespace_ - #include "ppport.h" - -The default namespace is C. - -=back - -The good thing is that most of the above can be checked by running -F on your source code. See the next section for -details. - -=head1 EXAMPLES - -To verify whether F is needed for your module, whether you -should make any changes to your code, and whether any special defines -should be used, F can be run as a Perl script to check your -source code. Simply say: - - perl ppport.h - -The result will usually be a list of patches suggesting changes -that should at least be acceptable, if not necessarily the most -efficient solution, or a fix for all possible problems. - -If you know that your XS module uses features only available in -newer Perl releases, if you're aware that it uses C++ comments, -and if you want all suggestions as a single patch file, you could -use something like this: - - perl ppport.h --compat-version=5.6.0 --cplusplus --patch=test.diff - -If you only want your code to be scanned without any suggestions -for changes, use: - - perl ppport.h --nochanges - -You can specify a different C program or options, using -the C<--diff> option: - - perl ppport.h --diff='diff -C 10' - -This would output context diffs with 10 lines of context. - -If you want to create patched copies of your files instead, use: - - perl ppport.h --copy=.new - -To display portability information for the C function, -use: - - perl ppport.h --api-info=newSVpvn - -Since the argument to C<--api-info> can be a regular expression, -you can use - - perl ppport.h --api-info=/_nomg$/ - -to display portability information for all C<_nomg> functions or - - perl ppport.h --api-info=/./ - -to display information for all known API elements. - -=head1 BUGS - -If this version of F is causing failure during -the compilation of this module, please check if newer versions -of either this module or C are available on CPAN -before sending a bug report. - -If F was generated using the latest version of -C and is causing failure of this module, please -file a bug report using the CPAN Request Tracker at L. - -Please include the following information: - -=over 4 - -=item 1. - -The complete output from running "perl -V" - -=item 2. - -This file. - -=item 3. - -The name and version of the module you were trying to build. - -=item 4. - -A full log of the build that failed. - -=item 5. - -Any other information that you think could be relevant. - -=back - -For the latest version of this code, please get the C -module from CPAN. - -=head1 COPYRIGHT - -Version 3.x, Copyright (c) 2004-2009, Marcus Holland-Moritz. - -Version 2.x, Copyright (C) 2001, Paul Marquess. - -Version 1.x, Copyright (C) 1999, Kenneth Albanowski. - -This program is free software; you can redistribute it and/or -modify it under the same terms as Perl itself. - -=head1 SEE ALSO - -See L. - -=cut - -use strict; - -# Disable broken TRIE-optimization -BEGIN { eval '${^RE_TRIE_MAXBUF} = -1' if $] >= 5.009004 && $] <= 5.009005 } - -my $VERSION = 3.19; - -my %opt = ( - quiet => 0, - diag => 1, - hints => 1, - changes => 1, - cplusplus => 0, - filter => 1, - strip => 0, - version => 0, -); - -my($ppport) = $0 =~ /([\w.]+)$/; -my $LF = '(?:\r\n|[\r\n])'; # line feed -my $HS = "[ \t]"; # horizontal whitespace - -# Never use C comments in this file! -my $ccs = '/'.'*'; -my $cce = '*'.'/'; -my $rccs = quotemeta $ccs; -my $rcce = quotemeta $cce; - -eval { - require Getopt::Long; - Getopt::Long::GetOptions(\%opt, qw( - help quiet diag! filter! hints! changes! cplusplus strip version - patch=s copy=s diff=s compat-version=s - list-provided list-unsupported api-info=s - )) or usage(); -}; - -if ($@ and grep /^-/, @ARGV) { - usage() if "@ARGV" =~ /^--?h(?:elp)?$/; - die "Getopt::Long not found. Please don't use any options.\n"; -} - -if ($opt{version}) { - print "This is $0 $VERSION.\n"; - exit 0; -} - -usage() if $opt{help}; -strip() if $opt{strip}; - -if (exists $opt{'compat-version'}) { - my($r,$v,$s) = eval { parse_version($opt{'compat-version'}) }; - if ($@) { - die "Invalid version number format: '$opt{'compat-version'}'\n"; - } - die "Only Perl 5 is supported\n" if $r != 5; - die "Invalid version number: $opt{'compat-version'}\n" if $v >= 1000 || $s >= 1000; - $opt{'compat-version'} = sprintf "%d.%03d%03d", $r, $v, $s; -} -else { - $opt{'compat-version'} = 5; -} - -my %API = map { /^(\w+)\|([^|]*)\|([^|]*)\|(\w*)$/ - ? ( $1 => { - ($2 ? ( base => $2 ) : ()), - ($3 ? ( todo => $3 ) : ()), - (index($4, 'v') >= 0 ? ( varargs => 1 ) : ()), - (index($4, 'p') >= 0 ? ( provided => 1 ) : ()), - (index($4, 'n') >= 0 ? ( nothxarg => 1 ) : ()), - } ) - : die "invalid spec: $_" } qw( -AvFILLp|5.004050||p -AvFILL||| -CLASS|||n -CPERLscope|5.005000||p -CX_CURPAD_SAVE||| -CX_CURPAD_SV||| -CopFILEAV|5.006000||p -CopFILEGV_set|5.006000||p -CopFILEGV|5.006000||p -CopFILESV|5.006000||p -CopFILE_set|5.006000||p -CopFILE|5.006000||p -CopSTASHPV_set|5.006000||p -CopSTASHPV|5.006000||p -CopSTASH_eq|5.006000||p -CopSTASH_set|5.006000||p -CopSTASH|5.006000||p -CopyD|5.009002||p -Copy||| -CvPADLIST||| -CvSTASH||| -CvWEAKOUTSIDE||| -DEFSV_set|5.011000||p -DEFSV|5.004050||p -END_EXTERN_C|5.005000||p -ENTER||| -ERRSV|5.004050||p -EXTEND||| -EXTERN_C|5.005000||p -F0convert|||n -FREETMPS||| -GIMME_V||5.004000|n -GIMME|||n -GROK_NUMERIC_RADIX|5.007002||p -G_ARRAY||| -G_DISCARD||| -G_EVAL||| -G_METHOD|5.006001||p -G_NOARGS||| -G_SCALAR||| -G_VOID||5.004000| -GetVars||| -GvSVn|5.009003||p -GvSV||| -Gv_AMupdate||| -HEf_SVKEY||5.004000| -HeHASH||5.004000| -HeKEY||5.004000| -HeKLEN||5.004000| -HePV||5.004000| -HeSVKEY_force||5.004000| -HeSVKEY_set||5.004000| -HeSVKEY||5.004000| -HeUTF8||5.011000| -HeVAL||5.004000| -HvNAMELEN_get|5.009003||p -HvNAME_get|5.009003||p -HvNAME||| -INT2PTR|5.006000||p -IN_LOCALE_COMPILETIME|5.007002||p -IN_LOCALE_RUNTIME|5.007002||p -IN_LOCALE|5.007002||p -IN_PERL_COMPILETIME|5.008001||p -IS_NUMBER_GREATER_THAN_UV_MAX|5.007002||p -IS_NUMBER_INFINITY|5.007002||p -IS_NUMBER_IN_UV|5.007002||p -IS_NUMBER_NAN|5.007003||p -IS_NUMBER_NEG|5.007002||p -IS_NUMBER_NOT_INT|5.007002||p -IVSIZE|5.006000||p -IVTYPE|5.006000||p -IVdf|5.006000||p -LEAVE||| -LVRET||| -MARK||| -MULTICALL||5.011000| -MY_CXT_CLONE|5.009002||p -MY_CXT_INIT|5.007003||p -MY_CXT|5.007003||p -MoveD|5.009002||p -Move||| -NOOP|5.005000||p -NUM2PTR|5.006000||p -NVTYPE|5.006000||p -NVef|5.006001||p -NVff|5.006001||p -NVgf|5.006001||p -Newxc|5.009003||p -Newxz|5.009003||p -Newx|5.009003||p -Nullav||| -Nullch||| -Nullcv||| -Nullhv||| -Nullsv||| -ORIGMARK||| -PAD_BASE_SV||| -PAD_CLONE_VARS||| -PAD_COMPNAME_FLAGS||| -PAD_COMPNAME_GEN_set||| -PAD_COMPNAME_GEN||| -PAD_COMPNAME_OURSTASH||| -PAD_COMPNAME_PV||| -PAD_COMPNAME_TYPE||| -PAD_DUP||| -PAD_RESTORE_LOCAL||| -PAD_SAVE_LOCAL||| -PAD_SAVE_SETNULLPAD||| -PAD_SETSV||| -PAD_SET_CUR_NOSAVE||| -PAD_SET_CUR||| -PAD_SVl||| -PAD_SV||| -PERLIO_FUNCS_CAST|5.009003||p -PERLIO_FUNCS_DECL|5.009003||p -PERL_ABS|5.008001||p -PERL_BCDVERSION|5.011000||p -PERL_GCC_BRACE_GROUPS_FORBIDDEN|5.008001||p -PERL_HASH|5.004000||p -PERL_INT_MAX|5.004000||p -PERL_INT_MIN|5.004000||p -PERL_LONG_MAX|5.004000||p -PERL_LONG_MIN|5.004000||p -PERL_MAGIC_arylen|5.007002||p -PERL_MAGIC_backref|5.007002||p -PERL_MAGIC_bm|5.007002||p -PERL_MAGIC_collxfrm|5.007002||p -PERL_MAGIC_dbfile|5.007002||p -PERL_MAGIC_dbline|5.007002||p -PERL_MAGIC_defelem|5.007002||p -PERL_MAGIC_envelem|5.007002||p -PERL_MAGIC_env|5.007002||p -PERL_MAGIC_ext|5.007002||p -PERL_MAGIC_fm|5.007002||p -PERL_MAGIC_glob|5.011000||p -PERL_MAGIC_isaelem|5.007002||p -PERL_MAGIC_isa|5.007002||p -PERL_MAGIC_mutex|5.011000||p -PERL_MAGIC_nkeys|5.007002||p -PERL_MAGIC_overload_elem|5.007002||p -PERL_MAGIC_overload_table|5.007002||p -PERL_MAGIC_overload|5.007002||p -PERL_MAGIC_pos|5.007002||p -PERL_MAGIC_qr|5.007002||p -PERL_MAGIC_regdata|5.007002||p -PERL_MAGIC_regdatum|5.007002||p -PERL_MAGIC_regex_global|5.007002||p -PERL_MAGIC_shared_scalar|5.007003||p -PERL_MAGIC_shared|5.007003||p -PERL_MAGIC_sigelem|5.007002||p -PERL_MAGIC_sig|5.007002||p -PERL_MAGIC_substr|5.007002||p -PERL_MAGIC_sv|5.007002||p -PERL_MAGIC_taint|5.007002||p -PERL_MAGIC_tiedelem|5.007002||p -PERL_MAGIC_tiedscalar|5.007002||p -PERL_MAGIC_tied|5.007002||p -PERL_MAGIC_utf8|5.008001||p -PERL_MAGIC_uvar_elem|5.007003||p -PERL_MAGIC_uvar|5.007002||p -PERL_MAGIC_vec|5.007002||p -PERL_MAGIC_vstring|5.008001||p -PERL_PV_ESCAPE_ALL|5.009004||p -PERL_PV_ESCAPE_FIRSTCHAR|5.009004||p -PERL_PV_ESCAPE_NOBACKSLASH|5.009004||p -PERL_PV_ESCAPE_NOCLEAR|5.009004||p -PERL_PV_ESCAPE_QUOTE|5.009004||p -PERL_PV_ESCAPE_RE|5.009005||p -PERL_PV_ESCAPE_UNI_DETECT|5.009004||p -PERL_PV_ESCAPE_UNI|5.009004||p -PERL_PV_PRETTY_DUMP|5.009004||p -PERL_PV_PRETTY_ELLIPSES|5.010000||p -PERL_PV_PRETTY_LTGT|5.009004||p -PERL_PV_PRETTY_NOCLEAR|5.010000||p -PERL_PV_PRETTY_QUOTE|5.009004||p -PERL_PV_PRETTY_REGPROP|5.009004||p -PERL_QUAD_MAX|5.004000||p -PERL_QUAD_MIN|5.004000||p -PERL_REVISION|5.006000||p -PERL_SCAN_ALLOW_UNDERSCORES|5.007003||p -PERL_SCAN_DISALLOW_PREFIX|5.007003||p -PERL_SCAN_GREATER_THAN_UV_MAX|5.007003||p -PERL_SCAN_SILENT_ILLDIGIT|5.008001||p -PERL_SHORT_MAX|5.004000||p -PERL_SHORT_MIN|5.004000||p -PERL_SIGNALS_UNSAFE_FLAG|5.008001||p -PERL_SUBVERSION|5.006000||p -PERL_SYS_INIT3||5.006000| -PERL_SYS_INIT||| -PERL_SYS_TERM||5.011000| -PERL_UCHAR_MAX|5.004000||p -PERL_UCHAR_MIN|5.004000||p -PERL_UINT_MAX|5.004000||p -PERL_UINT_MIN|5.004000||p -PERL_ULONG_MAX|5.004000||p -PERL_ULONG_MIN|5.004000||p -PERL_UNUSED_ARG|5.009003||p -PERL_UNUSED_CONTEXT|5.009004||p -PERL_UNUSED_DECL|5.007002||p -PERL_UNUSED_VAR|5.007002||p -PERL_UQUAD_MAX|5.004000||p -PERL_UQUAD_MIN|5.004000||p -PERL_USE_GCC_BRACE_GROUPS|5.009004||p -PERL_USHORT_MAX|5.004000||p -PERL_USHORT_MIN|5.004000||p -PERL_VERSION|5.006000||p -PL_DBsignal|5.005000||p -PL_DBsingle|||pn -PL_DBsub|||pn -PL_DBtrace|||pn -PL_Sv|5.005000||p -PL_bufend|5.011000||p -PL_bufptr|5.011000||p -PL_compiling|5.004050||p -PL_copline|5.011000||p -PL_curcop|5.004050||p -PL_curstash|5.004050||p -PL_debstash|5.004050||p -PL_defgv|5.004050||p -PL_diehook|5.004050||p -PL_dirty|5.004050||p -PL_dowarn|||pn -PL_errgv|5.004050||p -PL_error_count|5.011000||p -PL_expect|5.011000||p -PL_hexdigit|5.005000||p -PL_hints|5.005000||p -PL_in_my_stash|5.011000||p -PL_in_my|5.011000||p -PL_last_in_gv|||n -PL_laststatval|5.005000||p -PL_lex_state|5.011000||p -PL_lex_stuff|5.011000||p -PL_linestr|5.011000||p -PL_modglobal||5.005000|n -PL_na|5.004050||pn -PL_no_modify|5.006000||p -PL_ofsgv|||n -PL_parser|5.009005||p -PL_perl_destruct_level|5.004050||p -PL_perldb|5.004050||p -PL_ppaddr|5.006000||p -PL_rsfp_filters|5.004050||p -PL_rsfp|5.004050||p -PL_rs|||n -PL_signals|5.008001||p -PL_stack_base|5.004050||p -PL_stack_sp|5.004050||p -PL_statcache|5.005000||p -PL_stdingv|5.004050||p -PL_sv_arenaroot|5.004050||p -PL_sv_no|5.004050||pn -PL_sv_undef|5.004050||pn -PL_sv_yes|5.004050||pn -PL_tainted|5.004050||p -PL_tainting|5.004050||p -PL_tokenbuf|5.011000||p -POP_MULTICALL||5.011000| -POPi|||n -POPl|||n -POPn|||n -POPpbytex||5.007001|n -POPpx||5.005030|n -POPp|||n -POPs|||n -PTR2IV|5.006000||p -PTR2NV|5.006000||p -PTR2UV|5.006000||p -PTR2nat|5.009003||p -PTR2ul|5.007001||p -PTRV|5.006000||p -PUSHMARK||| -PUSH_MULTICALL||5.011000| -PUSHi||| -PUSHmortal|5.009002||p -PUSHn||| -PUSHp||| -PUSHs||| -PUSHu|5.004000||p -PUTBACK||| -PerlIO_clearerr||5.007003| -PerlIO_close||5.007003| -PerlIO_context_layers||5.009004| -PerlIO_eof||5.007003| -PerlIO_error||5.007003| -PerlIO_fileno||5.007003| -PerlIO_fill||5.007003| -PerlIO_flush||5.007003| -PerlIO_get_base||5.007003| -PerlIO_get_bufsiz||5.007003| -PerlIO_get_cnt||5.007003| -PerlIO_get_ptr||5.007003| -PerlIO_read||5.007003| -PerlIO_seek||5.007003| -PerlIO_set_cnt||5.007003| -PerlIO_set_ptrcnt||5.007003| -PerlIO_setlinebuf||5.007003| -PerlIO_stderr||5.007003| -PerlIO_stdin||5.007003| -PerlIO_stdout||5.007003| -PerlIO_tell||5.007003| -PerlIO_unread||5.007003| -PerlIO_write||5.007003| -Perl_signbit||5.009005|n -PoisonFree|5.009004||p -PoisonNew|5.009004||p -PoisonWith|5.009004||p -Poison|5.008000||p -RETVAL|||n -Renewc||| -Renew||| -SAVECLEARSV||| -SAVECOMPPAD||| -SAVEPADSV||| -SAVETMPS||| -SAVE_DEFSV|5.004050||p -SPAGAIN||| -SP||| -START_EXTERN_C|5.005000||p -START_MY_CXT|5.007003||p -STMT_END|||p -STMT_START|||p -STR_WITH_LEN|5.009003||p -ST||| -SV_CONST_RETURN|5.009003||p -SV_COW_DROP_PV|5.008001||p -SV_COW_SHARED_HASH_KEYS|5.009005||p -SV_GMAGIC|5.007002||p -SV_HAS_TRAILING_NUL|5.009004||p -SV_IMMEDIATE_UNREF|5.007001||p -SV_MUTABLE_RETURN|5.009003||p -SV_NOSTEAL|5.009002||p -SV_SMAGIC|5.009003||p -SV_UTF8_NO_ENCODING|5.008001||p -SVfARG|5.009005||p -SVf_UTF8|5.006000||p -SVf|5.006000||p -SVt_IV||| -SVt_NV||| -SVt_PVAV||| -SVt_PVCV||| -SVt_PVHV||| -SVt_PVMG||| -SVt_PV||| -Safefree||| -Slab_Alloc||| -Slab_Free||| -Slab_to_rw||| -StructCopy||| -SvCUR_set||| -SvCUR||| -SvEND||| -SvGAMAGIC||5.006001| -SvGETMAGIC|5.004050||p -SvGROW||| -SvIOK_UV||5.006000| -SvIOK_notUV||5.006000| -SvIOK_off||| -SvIOK_only_UV||5.006000| -SvIOK_only||| -SvIOK_on||| -SvIOKp||| -SvIOK||| -SvIVX||| -SvIV_nomg|5.009001||p -SvIV_set||| -SvIVx||| -SvIV||| -SvIsCOW_shared_hash||5.008003| -SvIsCOW||5.008003| -SvLEN_set||| -SvLEN||| -SvLOCK||5.007003| -SvMAGIC_set|5.009003||p -SvNIOK_off||| -SvNIOKp||| -SvNIOK||| -SvNOK_off||| -SvNOK_only||| -SvNOK_on||| -SvNOKp||| -SvNOK||| -SvNVX||| -SvNV_set||| -SvNVx||| -SvNV||| -SvOK||| -SvOOK_offset||5.011000| -SvOOK||| -SvPOK_off||| -SvPOK_only_UTF8||5.006000| -SvPOK_only||| -SvPOK_on||| -SvPOKp||| -SvPOK||| -SvPVX_const|5.009003||p -SvPVX_mutable|5.009003||p -SvPVX||| -SvPV_const|5.009003||p -SvPV_flags_const_nolen|5.009003||p -SvPV_flags_const|5.009003||p -SvPV_flags_mutable|5.009003||p -SvPV_flags|5.007002||p -SvPV_force_flags_mutable|5.009003||p -SvPV_force_flags_nolen|5.009003||p -SvPV_force_flags|5.007002||p -SvPV_force_mutable|5.009003||p -SvPV_force_nolen|5.009003||p -SvPV_force_nomg_nolen|5.009003||p -SvPV_force_nomg|5.007002||p -SvPV_force|||p -SvPV_mutable|5.009003||p -SvPV_nolen_const|5.009003||p -SvPV_nolen|5.006000||p -SvPV_nomg_const_nolen|5.009003||p -SvPV_nomg_const|5.009003||p -SvPV_nomg|5.007002||p -SvPV_renew|5.009003||p -SvPV_set||| -SvPVbyte_force||5.009002| -SvPVbyte_nolen||5.006000| -SvPVbytex_force||5.006000| -SvPVbytex||5.006000| -SvPVbyte|5.006000||p -SvPVutf8_force||5.006000| -SvPVutf8_nolen||5.006000| -SvPVutf8x_force||5.006000| -SvPVutf8x||5.006000| -SvPVutf8||5.006000| -SvPVx||| -SvPV||| -SvREFCNT_dec||| -SvREFCNT_inc_NN|5.009004||p -SvREFCNT_inc_simple_NN|5.009004||p -SvREFCNT_inc_simple_void_NN|5.009004||p -SvREFCNT_inc_simple_void|5.009004||p -SvREFCNT_inc_simple|5.009004||p -SvREFCNT_inc_void_NN|5.009004||p -SvREFCNT_inc_void|5.009004||p -SvREFCNT_inc|||p -SvREFCNT||| -SvROK_off||| -SvROK_on||| -SvROK||| -SvRV_set|5.009003||p -SvRV||| -SvRXOK||5.009005| -SvRX||5.009005| -SvSETMAGIC||| -SvSHARED_HASH|5.009003||p -SvSHARE||5.007003| -SvSTASH_set|5.009003||p -SvSTASH||| -SvSetMagicSV_nosteal||5.004000| -SvSetMagicSV||5.004000| -SvSetSV_nosteal||5.004000| -SvSetSV||| -SvTAINTED_off||5.004000| -SvTAINTED_on||5.004000| -SvTAINTED||5.004000| -SvTAINT||| -SvTRUE||| -SvTYPE||| -SvUNLOCK||5.007003| -SvUOK|5.007001|5.006000|p -SvUPGRADE||| -SvUTF8_off||5.006000| -SvUTF8_on||5.006000| -SvUTF8||5.006000| -SvUVXx|5.004000||p -SvUVX|5.004000||p -SvUV_nomg|5.009001||p -SvUV_set|5.009003||p -SvUVx|5.004000||p -SvUV|5.004000||p -SvVOK||5.008001| -SvVSTRING_mg|5.009004||p -THIS|||n -UNDERBAR|5.009002||p -UTF8_MAXBYTES|5.009002||p -UVSIZE|5.006000||p -UVTYPE|5.006000||p -UVXf|5.007001||p -UVof|5.006000||p -UVuf|5.006000||p -UVxf|5.006000||p -WARN_ALL|5.006000||p -WARN_AMBIGUOUS|5.006000||p -WARN_ASSERTIONS|5.011000||p -WARN_BAREWORD|5.006000||p -WARN_CLOSED|5.006000||p -WARN_CLOSURE|5.006000||p -WARN_DEBUGGING|5.006000||p -WARN_DEPRECATED|5.006000||p -WARN_DIGIT|5.006000||p -WARN_EXEC|5.006000||p -WARN_EXITING|5.006000||p -WARN_GLOB|5.006000||p -WARN_INPLACE|5.006000||p -WARN_INTERNAL|5.006000||p -WARN_IO|5.006000||p -WARN_LAYER|5.008000||p -WARN_MALLOC|5.006000||p -WARN_MISC|5.006000||p -WARN_NEWLINE|5.006000||p -WARN_NUMERIC|5.006000||p -WARN_ONCE|5.006000||p -WARN_OVERFLOW|5.006000||p -WARN_PACK|5.006000||p -WARN_PARENTHESIS|5.006000||p -WARN_PIPE|5.006000||p -WARN_PORTABLE|5.006000||p -WARN_PRECEDENCE|5.006000||p -WARN_PRINTF|5.006000||p -WARN_PROTOTYPE|5.006000||p -WARN_QW|5.006000||p -WARN_RECURSION|5.006000||p -WARN_REDEFINE|5.006000||p -WARN_REGEXP|5.006000||p -WARN_RESERVED|5.006000||p -WARN_SEMICOLON|5.006000||p -WARN_SEVERE|5.006000||p -WARN_SIGNAL|5.006000||p -WARN_SUBSTR|5.006000||p -WARN_SYNTAX|5.006000||p -WARN_TAINT|5.006000||p -WARN_THREADS|5.008000||p -WARN_UNINITIALIZED|5.006000||p -WARN_UNOPENED|5.006000||p -WARN_UNPACK|5.006000||p -WARN_UNTIE|5.006000||p -WARN_UTF8|5.006000||p -WARN_VOID|5.006000||p -XCPT_CATCH|5.009002||p -XCPT_RETHROW|5.009002||p -XCPT_TRY_END|5.009002||p -XCPT_TRY_START|5.009002||p -XPUSHi||| -XPUSHmortal|5.009002||p -XPUSHn||| -XPUSHp||| -XPUSHs||| -XPUSHu|5.004000||p -XSPROTO|5.010000||p -XSRETURN_EMPTY||| -XSRETURN_IV||| -XSRETURN_NO||| -XSRETURN_NV||| -XSRETURN_PV||| -XSRETURN_UNDEF||| -XSRETURN_UV|5.008001||p -XSRETURN_YES||| -XSRETURN|||p -XST_mIV||| -XST_mNO||| -XST_mNV||| -XST_mPV||| -XST_mUNDEF||| -XST_mUV|5.008001||p -XST_mYES||| -XS_VERSION_BOOTCHECK||| -XS_VERSION||| -XSprePUSH|5.006000||p -XS||| -ZeroD|5.009002||p -Zero||| -_aMY_CXT|5.007003||p -_pMY_CXT|5.007003||p -aMY_CXT_|5.007003||p -aMY_CXT|5.007003||p -aTHXR_|5.011000||p -aTHXR|5.011000||p -aTHX_|5.006000||p -aTHX|5.006000||p -add_data|||n -addmad||| -allocmy||| -amagic_call||| -amagic_cmp_locale||| -amagic_cmp||| -amagic_i_ncmp||| -amagic_ncmp||| -any_dup||| -ao||| -append_elem||| -append_list||| -append_madprops||| -apply_attrs_my||| -apply_attrs_string||5.006001| -apply_attrs||| -apply||| -atfork_lock||5.007003|n -atfork_unlock||5.007003|n -av_arylen_p||5.009003| -av_clear||| -av_create_and_push||5.009005| -av_create_and_unshift_one||5.009005| -av_delete||5.006000| -av_exists||5.006000| -av_extend||| -av_fetch||| -av_fill||| -av_iter_p||5.011000| -av_len||| -av_make||| -av_pop||| -av_push||| -av_reify||| -av_shift||| -av_store||| -av_undef||| -av_unshift||| -ax|||n -bad_type||| -bind_match||| -block_end||| -block_gimme||5.004000| -block_start||| -boolSV|5.004000||p -boot_core_PerlIO||| -boot_core_UNIVERSAL||| -boot_core_mro||| -bytes_from_utf8||5.007001| -bytes_to_uni|||n -bytes_to_utf8||5.006001| -call_argv|5.006000||p -call_atexit||5.006000| -call_list||5.004000| -call_method|5.006000||p -call_pv|5.006000||p -call_sv|5.006000||p -calloc||5.007002|n -cando||| -cast_i32||5.006000| -cast_iv||5.006000| -cast_ulong||5.006000| -cast_uv||5.006000| -check_type_and_open||| -check_uni||| -checkcomma||| -checkposixcc||| -ckWARN|5.006000||p -ck_anoncode||| -ck_bitop||| -ck_concat||| -ck_defined||| -ck_delete||| -ck_die||| -ck_each||| -ck_eof||| -ck_eval||| -ck_exec||| -ck_exists||| -ck_exit||| -ck_ftst||| -ck_fun||| -ck_glob||| -ck_grep||| -ck_index||| -ck_join||| -ck_lfun||| -ck_listiob||| -ck_match||| -ck_method||| -ck_null||| -ck_open||| -ck_readline||| -ck_repeat||| -ck_require||| -ck_return||| -ck_rfun||| -ck_rvconst||| -ck_sassign||| -ck_select||| -ck_shift||| -ck_sort||| -ck_spair||| -ck_split||| -ck_subr||| -ck_substr||| -ck_svconst||| -ck_trunc||| -ck_unpack||| -ckwarn_d||5.009003| -ckwarn||5.009003| -cl_and|||n -cl_anything|||n -cl_init_zero|||n -cl_init|||n -cl_is_anything|||n -cl_or|||n -clear_placeholders||| -closest_cop||| -convert||| -cop_free||| -cr_textfilter||| -create_eval_scope||| -croak_nocontext|||vn -croak_xs_usage||5.011000| -croak|||v -csighandler||5.009003|n -curmad||| -custom_op_desc||5.007003| -custom_op_name||5.007003| -cv_ckproto_len||| -cv_clone||| -cv_const_sv||5.004000| -cv_dump||| -cv_undef||| -cx_dump||5.005000| -cx_dup||| -cxinc||| -dAXMARK|5.009003||p -dAX|5.007002||p -dITEMS|5.007002||p -dMARK||| -dMULTICALL||5.009003| -dMY_CXT_SV|5.007003||p -dMY_CXT|5.007003||p -dNOOP|5.006000||p -dORIGMARK||| -dSP||| -dTHR|5.004050||p -dTHXR|5.011000||p -dTHXa|5.006000||p -dTHXoa|5.006000||p -dTHX|5.006000||p -dUNDERBAR|5.009002||p -dVAR|5.009003||p -dXCPT|5.009002||p -dXSARGS||| -dXSI32||| -dXSTARG|5.006000||p -deb_curcv||| -deb_nocontext|||vn -deb_stack_all||| -deb_stack_n||| -debop||5.005000| -debprofdump||5.005000| -debprof||| -debstackptrs||5.007003| -debstack||5.007003| -debug_start_match||| -deb||5.007003|v -del_sv||| -delete_eval_scope||| -delimcpy||5.004000| -deprecate_old||| -deprecate||| -despatch_signals||5.007001| -destroy_matcher||| -die_nocontext|||vn -die_where||| -die|||v -dirp_dup||| -div128||| -djSP||| -do_aexec5||| -do_aexec||| -do_aspawn||| -do_binmode||5.004050| -do_chomp||| -do_chop||| -do_close||| -do_dump_pad||| -do_eof||| -do_exec3||| -do_execfree||| -do_exec||| -do_gv_dump||5.006000| -do_gvgv_dump||5.006000| -do_hv_dump||5.006000| -do_ipcctl||| -do_ipcget||| -do_join||| -do_kv||| -do_magic_dump||5.006000| -do_msgrcv||| -do_msgsnd||| -do_oddball||| -do_op_dump||5.006000| -do_op_xmldump||| -do_open9||5.006000| -do_openn||5.007001| -do_open||5.004000| -do_pmop_dump||5.006000| -do_pmop_xmldump||| -do_print||| -do_readline||| -do_seek||| -do_semop||| -do_shmio||| -do_smartmatch||| -do_spawn_nowait||| -do_spawn||| -do_sprintf||| -do_sv_dump||5.006000| -do_sysseek||| -do_tell||| -do_trans_complex_utf8||| -do_trans_complex||| -do_trans_count_utf8||| -do_trans_count||| -do_trans_simple_utf8||| -do_trans_simple||| -do_trans||| -do_vecget||| -do_vecset||| -do_vop||| -docatch||| -doeval||| -dofile||| -dofindlabel||| -doform||| -doing_taint||5.008001|n -dooneliner||| -doopen_pm||| -doparseform||| -dopoptoeval||| -dopoptogiven||| -dopoptolabel||| -dopoptoloop||| -dopoptosub_at||| -dopoptowhen||| -doref||5.009003| -dounwind||| -dowantarray||| -dump_all||5.006000| -dump_eval||5.006000| -dump_exec_pos||| -dump_fds||| -dump_form||5.006000| -dump_indent||5.006000|v -dump_mstats||| -dump_packsubs||5.006000| -dump_sub||5.006000| -dump_sv_child||| -dump_trie_interim_list||| -dump_trie_interim_table||| -dump_trie||| -dump_vindent||5.006000| -dumpuntil||| -dup_attrlist||| -emulate_cop_io||| -eval_pv|5.006000||p -eval_sv|5.006000||p -exec_failed||| -expect_number||| -fbm_compile||5.005000| -fbm_instr||5.005000| -feature_is_enabled||| -fetch_cop_label||5.011000| -filter_add||| -filter_del||| -filter_gets||| -filter_read||| -find_and_forget_pmops||| -find_array_subscript||| -find_beginning||| -find_byclass||| -find_hash_subscript||| -find_in_my_stash||| -find_runcv||5.008001| -find_rundefsvoffset||5.009002| -find_script||| -find_uninit_var||| -first_symbol|||n -fold_constants||| -forbid_setid||| -force_ident||| -force_list||| -force_next||| -force_version||| -force_word||| -forget_pmop||| -form_nocontext|||vn -form||5.004000|v -fp_dup||| -fprintf_nocontext|||vn -free_global_struct||| -free_tied_hv_pool||| -free_tmps||| -gen_constant_list||| -get_arena||| -get_aux_mg||| -get_av|5.006000||p -get_context||5.006000|n -get_cvn_flags||5.009005| -get_cv|5.006000||p -get_db_sub||| -get_debug_opts||| -get_hash_seed||| -get_hv|5.006000||p -get_isa_hash||| -get_mstats||| -get_no_modify||| -get_num||| -get_op_descs||5.005000| -get_op_names||5.005000| -get_opargs||| -get_ppaddr||5.006000| -get_re_arg||| -get_sv|5.006000||p -get_vtbl||5.005030| -getcwd_sv||5.007002| -getenv_len||| -glob_2number||| -glob_assign_glob||| -glob_assign_ref||| -gp_dup||| -gp_free||| -gp_ref||| -grok_bin|5.007003||p -grok_hex|5.007003||p -grok_number|5.007002||p -grok_numeric_radix|5.007002||p -grok_oct|5.007003||p -group_end||| -gv_AVadd||| -gv_HVadd||| -gv_IOadd||| -gv_SVadd||| -gv_autoload4||5.004000| -gv_check||| -gv_const_sv||5.009003| -gv_dump||5.006000| -gv_efullname3||5.004000| -gv_efullname4||5.006001| -gv_efullname||| -gv_ename||| -gv_fetchfile_flags||5.009005| -gv_fetchfile||| -gv_fetchmeth_autoload||5.007003| -gv_fetchmethod_autoload||5.004000| -gv_fetchmethod_flags||5.011000| -gv_fetchmethod||| -gv_fetchmeth||| -gv_fetchpvn_flags|5.009002||p -gv_fetchpvs|5.009004||p -gv_fetchpv||| -gv_fetchsv||5.009002| -gv_fullname3||5.004000| -gv_fullname4||5.006001| -gv_fullname||| -gv_get_super_pkg||| -gv_handler||5.007001| -gv_init_sv||| -gv_init||| -gv_name_set||5.009004| -gv_stashpvn|5.004000||p -gv_stashpvs|5.009003||p -gv_stashpv||| -gv_stashsv||| -he_dup||| -hek_dup||| -hfreeentries||| -hsplit||| -hv_assert||5.011000| -hv_auxinit|||n -hv_backreferences_p||| -hv_clear_placeholders||5.009001| -hv_clear||| -hv_common_key_len||5.010000| -hv_common||5.010000| -hv_copy_hints_hv||| -hv_delayfree_ent||5.004000| -hv_delete_common||| -hv_delete_ent||5.004000| -hv_delete||| -hv_eiter_p||5.009003| -hv_eiter_set||5.009003| -hv_exists_ent||5.004000| -hv_exists||| -hv_fetch_ent||5.004000| -hv_fetchs|5.009003||p -hv_fetch||| -hv_free_ent||5.004000| -hv_iterinit||| -hv_iterkeysv||5.004000| -hv_iterkey||| -hv_iternext_flags||5.008000| -hv_iternextsv||| -hv_iternext||| -hv_iterval||| -hv_kill_backrefs||| -hv_ksplit||5.004000| -hv_magic_check|||n -hv_magic||| -hv_name_set||5.009003| -hv_notallowed||| -hv_placeholders_get||5.009003| -hv_placeholders_p||5.009003| -hv_placeholders_set||5.009003| -hv_riter_p||5.009003| -hv_riter_set||5.009003| -hv_scalar||5.009001| -hv_store_ent||5.004000| -hv_store_flags||5.008000| -hv_stores|5.009004||p -hv_store||| -hv_undef||| -ibcmp_locale||5.004000| -ibcmp_utf8||5.007003| -ibcmp||| -incline||| -incpush_if_exists||| -incpush_use_sep||| -incpush||| -ingroup||| -init_argv_symbols||| -init_debugger||| -init_global_struct||| -init_i18nl10n||5.006000| -init_i18nl14n||5.006000| -init_ids||| -init_interp||| -init_main_stash||| -init_perllib||| -init_postdump_symbols||| -init_predump_symbols||| -init_stacks||5.005000| -init_tm||5.007002| -instr||| -intro_my||| -intuit_method||| -intuit_more||| -invert||| -io_close||| -isALNUMC|5.006000||p -isALNUM||| -isALPHA||| -isASCII|5.006000||p -isBLANK|5.006001||p -isCNTRL|5.006000||p -isDIGIT||| -isGRAPH|5.006000||p -isGV_with_GP|5.009004||p -isLOWER||| -isPRINT|5.004000||p -isPSXSPC|5.006001||p -isPUNCT|5.006000||p -isSPACE||| -isUPPER||| -isXDIGIT|5.006000||p -is_an_int||| -is_gv_magical_sv||| -is_handle_constructor|||n -is_list_assignment||| -is_lvalue_sub||5.007001| -is_uni_alnum_lc||5.006000| -is_uni_alnumc_lc||5.006000| -is_uni_alnumc||5.006000| -is_uni_alnum||5.006000| -is_uni_alpha_lc||5.006000| -is_uni_alpha||5.006000| -is_uni_ascii_lc||5.006000| -is_uni_ascii||5.006000| -is_uni_cntrl_lc||5.006000| -is_uni_cntrl||5.006000| -is_uni_digit_lc||5.006000| -is_uni_digit||5.006000| -is_uni_graph_lc||5.006000| -is_uni_graph||5.006000| -is_uni_idfirst_lc||5.006000| -is_uni_idfirst||5.006000| -is_uni_lower_lc||5.006000| -is_uni_lower||5.006000| -is_uni_print_lc||5.006000| -is_uni_print||5.006000| -is_uni_punct_lc||5.006000| -is_uni_punct||5.006000| -is_uni_space_lc||5.006000| -is_uni_space||5.006000| -is_uni_upper_lc||5.006000| -is_uni_upper||5.006000| -is_uni_xdigit_lc||5.006000| -is_uni_xdigit||5.006000| -is_utf8_alnumc||5.006000| -is_utf8_alnum||5.006000| -is_utf8_alpha||5.006000| -is_utf8_ascii||5.006000| -is_utf8_char_slow|||n -is_utf8_char||5.006000| -is_utf8_cntrl||5.006000| -is_utf8_common||| -is_utf8_digit||5.006000| -is_utf8_graph||5.006000| -is_utf8_idcont||5.008000| -is_utf8_idfirst||5.006000| -is_utf8_lower||5.006000| -is_utf8_mark||5.006000| -is_utf8_print||5.006000| -is_utf8_punct||5.006000| -is_utf8_space||5.006000| -is_utf8_string_loclen||5.009003| -is_utf8_string_loc||5.008001| -is_utf8_string||5.006001| -is_utf8_upper||5.006000| -is_utf8_xdigit||5.006000| -isa_lookup||| -items|||n -ix|||n -jmaybe||| -join_exact||| -keyword||| -leave_scope||| -lex_end||| -lex_start||| -linklist||| -listkids||| -list||| -load_module_nocontext|||vn -load_module|5.006000||pv -localize||| -looks_like_bool||| -looks_like_number||| -lop||| -mPUSHi|5.009002||p -mPUSHn|5.009002||p -mPUSHp|5.009002||p -mPUSHs|5.011000||p -mPUSHu|5.009002||p -mXPUSHi|5.009002||p -mXPUSHn|5.009002||p -mXPUSHp|5.009002||p -mXPUSHs|5.011000||p -mXPUSHu|5.009002||p -mad_free||| -madlex||| -madparse||| -magic_clear_all_env||| -magic_clearenv||| -magic_clearhint||| -magic_clearisa||| -magic_clearpack||| -magic_clearsig||| -magic_dump||5.006000| -magic_existspack||| -magic_freearylen_p||| -magic_freeovrld||| -magic_getarylen||| -magic_getdefelem||| -magic_getnkeys||| -magic_getpack||| -magic_getpos||| -magic_getsig||| -magic_getsubstr||| -magic_gettaint||| -magic_getuvar||| -magic_getvec||| -magic_get||| -magic_killbackrefs||| -magic_len||| -magic_methcall||| -magic_methpack||| -magic_nextpack||| -magic_regdata_cnt||| -magic_regdatum_get||| -magic_regdatum_set||| -magic_scalarpack||| -magic_set_all_env||| -magic_setamagic||| -magic_setarylen||| -magic_setcollxfrm||| -magic_setdbline||| -magic_setdefelem||| -magic_setenv||| -magic_sethint||| -magic_setisa||| -magic_setmglob||| -magic_setnkeys||| -magic_setpack||| -magic_setpos||| -magic_setregexp||| -magic_setsig||| -magic_setsubstr||| -magic_settaint||| -magic_setutf8||| -magic_setuvar||| -magic_setvec||| -magic_set||| -magic_sizepack||| -magic_wipepack||| -make_matcher||| -make_trie_failtable||| -make_trie||| -malloc_good_size|||n -malloced_size|||n -malloc||5.007002|n -markstack_grow||| -matcher_matches_sv||| -measure_struct||| -memEQ|5.004000||p -memNE|5.004000||p -mem_collxfrm||| -mem_log_common|||n -mess_alloc||| -mess_nocontext|||vn -mess||5.006000|v -method_common||| -mfree||5.007002|n -mg_clear||| -mg_copy||| -mg_dup||| -mg_find||| -mg_free||| -mg_get||| -mg_length||5.005000| -mg_localize||| -mg_magical||| -mg_set||| -mg_size||5.005000| -mini_mktime||5.007002| -missingterm||| -mode_from_discipline||| -modkids||| -mod||| -more_bodies||| -more_sv||| -moreswitches||| -mro_get_from_name||5.011000| -mro_get_linear_isa_dfs||| -mro_get_linear_isa||5.009005| -mro_get_private_data||5.011000| -mro_isa_changed_in||| -mro_meta_dup||| -mro_meta_init||| -mro_method_changed_in||5.009005| -mro_register||5.011000| -mro_set_mro||5.011000| -mro_set_private_data||5.011000| -mul128||| -mulexp10|||n -my_atof2||5.007002| -my_atof||5.006000| -my_attrs||| -my_bcopy|||n -my_betoh16|||n -my_betoh32|||n -my_betoh64|||n -my_betohi|||n -my_betohl|||n -my_betohs|||n -my_bzero|||n -my_chsize||| -my_clearenv||| -my_cxt_index||| -my_cxt_init||| -my_dirfd||5.009005| -my_exit_jump||| -my_exit||| -my_failure_exit||5.004000| -my_fflush_all||5.006000| -my_fork||5.007003|n -my_htobe16|||n -my_htobe32|||n -my_htobe64|||n -my_htobei|||n -my_htobel|||n -my_htobes|||n -my_htole16|||n -my_htole32|||n -my_htole64|||n -my_htolei|||n -my_htolel|||n -my_htoles|||n -my_htonl||| -my_kid||| -my_letoh16|||n -my_letoh32|||n -my_letoh64|||n -my_letohi|||n -my_letohl|||n -my_letohs|||n -my_lstat||| -my_memcmp||5.004000|n -my_memset|||n -my_ntohl||| -my_pclose||5.004000| -my_popen_list||5.007001| -my_popen||5.004000| -my_setenv||| -my_snprintf|5.009004||pvn -my_socketpair||5.007003|n -my_sprintf|5.009003||pvn -my_stat||| -my_strftime||5.007002| -my_strlcat|5.009004||pn -my_strlcpy|5.009004||pn -my_swabn|||n -my_swap||| -my_unexec||| -my_vsnprintf||5.009004|n -need_utf8|||n -newANONATTRSUB||5.006000| -newANONHASH||| -newANONLIST||| -newANONSUB||| -newASSIGNOP||| -newATTRSUB||5.006000| -newAVREF||| -newAV||| -newBINOP||| -newCONDOP||| -newCONSTSUB|5.004050||p -newCVREF||| -newDEFSVOP||| -newFORM||| -newFOROP||| -newGIVENOP||5.009003| -newGIVWHENOP||| -newGP||| -newGVOP||| -newGVREF||| -newGVgen||| -newHVREF||| -newHVhv||5.005000| -newHV||| -newIO||| -newLISTOP||| -newLOGOP||| -newLOOPEX||| -newLOOPOP||| -newMADPROP||| -newMADsv||| -newMYSUB||| -newNULLLIST||| -newOP||| -newPADOP||| -newPMOP||| -newPROG||| -newPVOP||| -newRANGE||| -newRV_inc|5.004000||p -newRV_noinc|5.004000||p -newRV||| -newSLICEOP||| -newSTATEOP||| -newSUB||| -newSVOP||| -newSVREF||| -newSV_type|5.009005||p -newSVhek||5.009003| -newSViv||| -newSVnv||| -newSVpvf_nocontext|||vn -newSVpvf||5.004000|v -newSVpvn_flags|5.011000||p -newSVpvn_share|5.007001||p -newSVpvn_utf8|5.011000||p -newSVpvn|5.004050||p -newSVpvs_flags|5.011000||p -newSVpvs_share||5.009003| -newSVpvs|5.009003||p -newSVpv||| -newSVrv||| -newSVsv||| -newSVuv|5.006000||p -newSV||| -newTOKEN||| -newUNOP||| -newWHENOP||5.009003| -newWHILEOP||5.009003| -newXS_flags||5.009004| -newXSproto||5.006000| -newXS||5.006000| -new_collate||5.006000| -new_constant||| -new_ctype||5.006000| -new_he||| -new_logop||| -new_numeric||5.006000| -new_stackinfo||5.005000| -new_version||5.009000| -new_warnings_bitfield||| -next_symbol||| -nextargv||| -nextchar||| -ninstr||| -no_bareword_allowed||| -no_fh_allowed||| -no_op||| -not_a_number||| -nothreadhook||5.008000| -nuke_stacks||| -num_overflow|||n -offer_nice_chunk||| -oopsAV||| -oopsHV||| -op_clear||| -op_const_sv||| -op_dump||5.006000| -op_free||| -op_getmad_weak||| -op_getmad||| -op_null||5.007002| -op_refcnt_dec||| -op_refcnt_inc||| -op_refcnt_lock||5.009002| -op_refcnt_unlock||5.009002| -op_xmldump||| -open_script||| -pMY_CXT_|5.007003||p -pMY_CXT|5.007003||p -pTHX_|5.006000||p -pTHX|5.006000||p -packWARN|5.007003||p -pack_cat||5.007003| -pack_rec||| -package||| -packlist||5.008001| -pad_add_anon||| -pad_add_name||| -pad_alloc||| -pad_block_start||| -pad_check_dup||| -pad_compname_type||| -pad_findlex||| -pad_findmy||| -pad_fixup_inner_anons||| -pad_free||| -pad_leavemy||| -pad_new||| -pad_peg|||n -pad_push||| -pad_reset||| -pad_setsv||| -pad_sv||5.011000| -pad_swipe||| -pad_tidy||| -pad_undef||| -parse_body||| -parse_unicode_opts||| -parser_dup||| -parser_free||| -path_is_absolute|||n -peep||| -pending_Slabs_to_ro||| -perl_alloc_using|||n -perl_alloc|||n -perl_clone_using|||n -perl_clone|||n -perl_construct|||n -perl_destruct||5.007003|n -perl_free|||n -perl_parse||5.006000|n -perl_run|||n -pidgone||| -pm_description||| -pmflag||| -pmop_dump||5.006000| -pmop_xmldump||| -pmruntime||| -pmtrans||| -pop_scope||| -pregcomp||5.009005| -pregexec||| -pregfree2||5.011000| -pregfree||| -prepend_elem||| -prepend_madprops||| -printbuf||| -printf_nocontext|||vn -process_special_blocks||| -ptr_table_clear||5.009005| -ptr_table_fetch||5.009005| -ptr_table_find|||n -ptr_table_free||5.009005| -ptr_table_new||5.009005| -ptr_table_split||5.009005| -ptr_table_store||5.009005| -push_scope||| -put_byte||| -pv_display|5.006000||p -pv_escape|5.009004||p -pv_pretty|5.009004||p -pv_uni_display||5.007003| -qerror||| -qsortsvu||| -re_compile||5.009005| -re_croak2||| -re_dup_guts||| -re_intuit_start||5.009005| -re_intuit_string||5.006000| -readpipe_override||| -realloc||5.007002|n -reentrant_free||| -reentrant_init||| -reentrant_retry|||vn -reentrant_size||| -ref_array_or_hash||| -refcounted_he_chain_2hv||| -refcounted_he_fetch||| -refcounted_he_free||| -refcounted_he_new_common||| -refcounted_he_new||| -refcounted_he_value||| -refkids||| -refto||| -ref||5.011000| -reg_check_named_buff_matched||| -reg_named_buff_all||5.009005| -reg_named_buff_exists||5.009005| -reg_named_buff_fetch||5.009005| -reg_named_buff_firstkey||5.009005| -reg_named_buff_iter||| -reg_named_buff_nextkey||5.009005| -reg_named_buff_scalar||5.009005| -reg_named_buff||| -reg_namedseq||| -reg_node||| -reg_numbered_buff_fetch||| -reg_numbered_buff_length||| -reg_numbered_buff_store||| -reg_qr_package||| -reg_recode||| -reg_scan_name||| -reg_skipcomment||| -reg_temp_copy||| -reganode||| -regatom||| -regbranch||| -regclass_swash||5.009004| -regclass||| -regcppop||| -regcppush||| -regcurly|||n -regdump_extflags||| -regdump||5.005000| -regdupe_internal||| -regexec_flags||5.005000| -regfree_internal||5.009005| -reghop3|||n -reghop4|||n -reghopmaybe3|||n -reginclass||| -reginitcolors||5.006000| -reginsert||| -regmatch||| -regnext||5.005000| -regpiece||| -regpposixcc||| -regprop||| -regrepeat||| -regtail_study||| -regtail||| -regtry||| -reguni||| -regwhite|||n -reg||| -repeatcpy||| -report_evil_fh||| -report_uninit||| -require_pv||5.006000| -require_tie_mod||| -restore_magic||| -rninstr||| -rsignal_restore||| -rsignal_save||| -rsignal_state||5.004000| -rsignal||5.004000| -run_body||| -run_user_filter||| -runops_debug||5.005000| -runops_standard||5.005000| -rvpv_dup||| -rxres_free||| -rxres_restore||| -rxres_save||| -safesyscalloc||5.006000|n -safesysfree||5.006000|n -safesysmalloc||5.006000|n -safesysrealloc||5.006000|n -same_dirent||| -save_I16||5.004000| -save_I32||| -save_I8||5.006000| -save_adelete||5.011000| -save_aelem||5.004050| -save_alloc||5.006000| -save_aptr||| -save_ary||| -save_bool||5.008001| -save_clearsv||| -save_delete||| -save_destructor_x||5.006000| -save_destructor||5.006000| -save_freeop||| -save_freepv||| -save_freesv||| -save_generic_pvref||5.006001| -save_generic_svref||5.005030| -save_gp||5.004000| -save_hash||| -save_hek_flags|||n -save_helem_flags||5.011000| -save_helem||5.004050| -save_hints||| -save_hptr||| -save_int||| -save_item||| -save_iv||5.005000| -save_lines||| -save_list||| -save_long||| -save_magic||| -save_mortalizesv||5.007001| -save_nogv||| -save_op||| -save_padsv_and_mortalize||5.011000| -save_pptr||| -save_pushi32ptr||| -save_pushptri32ptr||| -save_pushptrptr||| -save_pushptr||5.011000| -save_re_context||5.006000| -save_scalar_at||| -save_scalar||| -save_set_svflags||5.009000| -save_shared_pvref||5.007003| -save_sptr||| -save_svref||| -save_vptr||5.006000| -savepvn||| -savepvs||5.009003| -savepv||| -savesharedpvn||5.009005| -savesharedpv||5.007003| -savestack_grow_cnt||5.008001| -savestack_grow||| -savesvpv||5.009002| -sawparens||| -scalar_mod_type|||n -scalarboolean||| -scalarkids||| -scalarseq||| -scalarvoid||| -scalar||| -scan_bin||5.006000| -scan_commit||| -scan_const||| -scan_formline||| -scan_heredoc||| -scan_hex||| -scan_ident||| -scan_inputsymbol||| -scan_num||5.007001| -scan_oct||| -scan_pat||| -scan_str||| -scan_subst||| -scan_trans||| -scan_version||5.009001| -scan_vstring||5.009005| -scan_word||| -scope||| -screaminstr||5.005000| -search_const||| -seed||5.008001| -sequence_num||| -sequence_tail||| -sequence||| -set_context||5.006000|n -set_numeric_local||5.006000| -set_numeric_radix||5.006000| -set_numeric_standard||5.006000| -setdefout||| -share_hek_flags||| -share_hek||5.004000| -si_dup||| -sighandler|||n -simplify_sort||| -skipspace0||| -skipspace1||| -skipspace2||| -skipspace||| -softref2xv||| -sortcv_stacked||| -sortcv_xsub||| -sortcv||| -sortsv_flags||5.009003| -sortsv||5.007003| -space_join_names_mortal||| -ss_dup||| -stack_grow||| -start_force||| -start_glob||| -start_subparse||5.004000| -stashpv_hvname_match||5.011000| -stdize_locale||| -store_cop_label||| -strEQ||| -strGE||| -strGT||| -strLE||| -strLT||| -strNE||| -str_to_version||5.006000| -strip_return||| -strnEQ||| -strnNE||| -study_chunk||| -sub_crush_depth||| -sublex_done||| -sublex_push||| -sublex_start||| -sv_2bool||| -sv_2cv||| -sv_2io||| -sv_2iuv_common||| -sv_2iuv_non_preserve||| -sv_2iv_flags||5.009001| -sv_2iv||| -sv_2mortal||| -sv_2num||| -sv_2nv||| -sv_2pv_flags|5.007002||p -sv_2pv_nolen|5.006000||p -sv_2pvbyte_nolen|5.006000||p -sv_2pvbyte|5.006000||p -sv_2pvutf8_nolen||5.006000| -sv_2pvutf8||5.006000| -sv_2pv||| -sv_2uv_flags||5.009001| -sv_2uv|5.004000||p -sv_add_arena||| -sv_add_backref||| -sv_backoff||| -sv_bless||| -sv_cat_decode||5.008001| -sv_catpv_mg|5.004050||p -sv_catpvf_mg_nocontext|||pvn -sv_catpvf_mg|5.006000|5.004000|pv -sv_catpvf_nocontext|||vn -sv_catpvf||5.004000|v -sv_catpvn_flags||5.007002| -sv_catpvn_mg|5.004050||p -sv_catpvn_nomg|5.007002||p -sv_catpvn||| -sv_catpvs|5.009003||p -sv_catpv||| -sv_catsv_flags||5.007002| -sv_catsv_mg|5.004050||p -sv_catsv_nomg|5.007002||p -sv_catsv||| -sv_catxmlpvn||| -sv_catxmlsv||| -sv_chop||| -sv_clean_all||| -sv_clean_objs||| -sv_clear||| -sv_cmp_locale||5.004000| -sv_cmp||| -sv_collxfrm||| -sv_compile_2op||5.008001| -sv_copypv||5.007003| -sv_dec||| -sv_del_backref||| -sv_derived_from||5.004000| -sv_destroyable||5.010000| -sv_does||5.009004| -sv_dump||| -sv_dup_inc_multiple||| -sv_dup||| -sv_eq||| -sv_exp_grow||| -sv_force_normal_flags||5.007001| -sv_force_normal||5.006000| -sv_free2||| -sv_free_arenas||| -sv_free||| -sv_gets||5.004000| -sv_grow||| -sv_i_ncmp||| -sv_inc||| -sv_insert_flags||5.011000| -sv_insert||| -sv_isa||| -sv_isobject||| -sv_iv||5.005000| -sv_kill_backrefs||| -sv_len_utf8||5.006000| -sv_len||| -sv_magic_portable|5.011000|5.004000|p -sv_magicext||5.007003| -sv_magic||| -sv_mortalcopy||| -sv_ncmp||| -sv_newmortal||| -sv_newref||| -sv_nolocking||5.007003| -sv_nosharing||5.007003| -sv_nounlocking||| -sv_nv||5.005000| -sv_peek||5.005000| -sv_pos_b2u_midway||| -sv_pos_b2u||5.006000| -sv_pos_u2b_cached||| -sv_pos_u2b_forwards|||n -sv_pos_u2b_midway|||n -sv_pos_u2b||5.006000| -sv_pvbyten_force||5.006000| -sv_pvbyten||5.006000| -sv_pvbyte||5.006000| -sv_pvn_force_flags|5.007002||p -sv_pvn_force||| -sv_pvn_nomg|5.007003|5.005000|p -sv_pvn||5.005000| -sv_pvutf8n_force||5.006000| -sv_pvutf8n||5.006000| -sv_pvutf8||5.006000| -sv_pv||5.006000| -sv_recode_to_utf8||5.007003| -sv_reftype||| -sv_release_COW||| -sv_replace||| -sv_report_used||| -sv_reset||| -sv_rvweaken||5.006000| -sv_setiv_mg|5.004050||p -sv_setiv||| -sv_setnv_mg|5.006000||p -sv_setnv||| -sv_setpv_mg|5.004050||p -sv_setpvf_mg_nocontext|||pvn -sv_setpvf_mg|5.006000|5.004000|pv -sv_setpvf_nocontext|||vn -sv_setpvf||5.004000|v -sv_setpviv_mg||5.008001| -sv_setpviv||5.008001| -sv_setpvn_mg|5.004050||p -sv_setpvn||| -sv_setpvs|5.009004||p -sv_setpv||| -sv_setref_iv||| -sv_setref_nv||| -sv_setref_pvn||| -sv_setref_pv||| -sv_setref_uv||5.007001| -sv_setsv_cow||| -sv_setsv_flags||5.007002| -sv_setsv_mg|5.004050||p -sv_setsv_nomg|5.007002||p -sv_setsv||| -sv_setuv_mg|5.004050||p -sv_setuv|5.004000||p -sv_tainted||5.004000| -sv_taint||5.004000| -sv_true||5.005000| -sv_unglob||| -sv_uni_display||5.007003| -sv_unmagic||| -sv_unref_flags||5.007001| -sv_unref||| -sv_untaint||5.004000| -sv_upgrade||| -sv_usepvn_flags||5.009004| -sv_usepvn_mg|5.004050||p -sv_usepvn||| -sv_utf8_decode||5.006000| -sv_utf8_downgrade||5.006000| -sv_utf8_encode||5.006000| -sv_utf8_upgrade_flags_grow||5.011000| -sv_utf8_upgrade_flags||5.007002| -sv_utf8_upgrade_nomg||5.007002| -sv_utf8_upgrade||5.007001| -sv_uv|5.005000||p -sv_vcatpvf_mg|5.006000|5.004000|p -sv_vcatpvfn||5.004000| -sv_vcatpvf|5.006000|5.004000|p -sv_vsetpvf_mg|5.006000|5.004000|p -sv_vsetpvfn||5.004000| -sv_vsetpvf|5.006000|5.004000|p -sv_xmlpeek||| -svtype||| -swallow_bom||| -swap_match_buff||| -swash_fetch||5.007002| -swash_get||| -swash_init||5.006000| -sys_init3||5.010000|n -sys_init||5.010000|n -sys_intern_clear||| -sys_intern_dup||| -sys_intern_init||| -sys_term||5.010000|n -taint_env||| -taint_proper||| -tmps_grow||5.006000| -toLOWER||| -toUPPER||| -to_byte_substr||| -to_uni_fold||5.007003| -to_uni_lower_lc||5.006000| -to_uni_lower||5.007003| -to_uni_title_lc||5.006000| -to_uni_title||5.007003| -to_uni_upper_lc||5.006000| -to_uni_upper||5.007003| -to_utf8_case||5.007003| -to_utf8_fold||5.007003| -to_utf8_lower||5.007003| -to_utf8_substr||| -to_utf8_title||5.007003| -to_utf8_upper||5.007003| -token_free||| -token_getmad||| -tokenize_use||| -tokeq||| -tokereport||| -too_few_arguments||| -too_many_arguments||| -uiv_2buf|||n -unlnk||| -unpack_rec||| -unpack_str||5.007003| -unpackstring||5.008001| -unshare_hek_or_pvn||| -unshare_hek||| -unsharepvn||5.004000| -unwind_handler_stack||| -update_debugger_info||| -upg_version||5.009005| -usage||| -utf16_to_utf8_reversed||5.006001| -utf16_to_utf8||5.006001| -utf8_distance||5.006000| -utf8_hop||5.006000| -utf8_length||5.007001| -utf8_mg_pos_cache_update||| -utf8_to_bytes||5.006001| -utf8_to_uvchr||5.007001| -utf8_to_uvuni||5.007001| -utf8n_to_uvchr||| -utf8n_to_uvuni||5.007001| -utilize||| -uvchr_to_utf8_flags||5.007003| -uvchr_to_utf8||| -uvuni_to_utf8_flags||5.007003| -uvuni_to_utf8||5.007001| -validate_suid||| -varname||| -vcmp||5.009000| -vcroak||5.006000| -vdeb||5.007003| -vdie_common||| -vdie_croak_common||| -vdie||| -vform||5.006000| -visit||| -vivify_defelem||| -vivify_ref||| -vload_module|5.006000||p -vmess||5.006000| -vnewSVpvf|5.006000|5.004000|p -vnormal||5.009002| -vnumify||5.009000| -vstringify||5.009000| -vverify||5.009003| -vwarner||5.006000| -vwarn||5.006000| -wait4pid||| -warn_nocontext|||vn -warner_nocontext|||vn -warner|5.006000|5.004000|pv -warn|||v -watch||| -whichsig||| -write_no_mem||| -write_to_stderr||| -xmldump_all||| -xmldump_attr||| -xmldump_eval||| -xmldump_form||| -xmldump_indent|||v -xmldump_packsubs||| -xmldump_sub||| -xmldump_vindent||| -yyerror||| -yylex||| -yyparse||| -yywarn||| -); - -if (exists $opt{'list-unsupported'}) { - my $f; - for $f (sort { lc $a cmp lc $b } keys %API) { - next unless $API{$f}{todo}; - print "$f ", '.'x(40-length($f)), " ", format_version($API{$f}{todo}), "\n"; - } - exit 0; -} - -# Scan for possible replacement candidates - -my(%replace, %need, %hints, %warnings, %depends); -my $replace = 0; -my($hint, $define, $function); - -sub find_api -{ - my $code = shift; - $code =~ s{ - / (?: \*[^*]*\*+(?:[^$ccs][^*]*\*+)* / | /[^\r\n]*) - | "[^"\\]*(?:\\.[^"\\]*)*" - | '[^'\\]*(?:\\.[^'\\]*)*' }{}egsx; - grep { exists $API{$_} } $code =~ /(\w+)/mg; -} - -while () { - if ($hint) { - my $h = $hint->[0] eq 'Hint' ? \%hints : \%warnings; - if (m{^\s*\*\s(.*?)\s*$}) { - for (@{$hint->[1]}) { - $h->{$_} ||= ''; # suppress warning with older perls - $h->{$_} .= "$1\n"; - } - } - else { undef $hint } - } - - $hint = [$1, [split /,?\s+/, $2]] - if m{^\s*$rccs\s+(Hint|Warning):\s+(\w+(?:,?\s+\w+)*)\s*$}; - - if ($define) { - if ($define->[1] =~ /\\$/) { - $define->[1] .= $_; - } - else { - if (exists $API{$define->[0]} && $define->[1] !~ /^DPPP_\(/) { - my @n = find_api($define->[1]); - push @{$depends{$define->[0]}}, @n if @n - } - undef $define; - } - } - - $define = [$1, $2] if m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(.*)}; - - if ($function) { - if (/^}/) { - if (exists $API{$function->[0]}) { - my @n = find_api($function->[1]); - push @{$depends{$function->[0]}}, @n if @n - } - undef $function; - } - else { - $function->[1] .= $_; - } - } - - $function = [$1, ''] if m{^DPPP_\(my_(\w+)\)}; - - $replace = $1 if m{^\s*$rccs\s+Replace:\s+(\d+)\s+$rcce\s*$}; - $replace{$2} = $1 if $replace and m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(\w+)}; - $replace{$2} = $1 if m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(\w+).*$rccs\s+Replace\s+$rcce}; - $replace{$1} = $2 if m{^\s*$rccs\s+Replace (\w+) with (\w+)\s+$rcce\s*$}; - - if (m{^\s*$rccs\s+(\w+(\s*,\s*\w+)*)\s+depends\s+on\s+(\w+(\s*,\s*\w+)*)\s+$rcce\s*$}) { - my @deps = map { s/\s+//g; $_ } split /,/, $3; - my $d; - for $d (map { s/\s+//g; $_ } split /,/, $1) { - push @{$depends{$d}}, @deps; - } - } - - $need{$1} = 1 if m{^#if\s+defined\(NEED_(\w+)(?:_GLOBAL)?\)}; -} - -for (values %depends) { - my %s; - $_ = [sort grep !$s{$_}++, @$_]; -} - -if (exists $opt{'api-info'}) { - my $f; - my $count = 0; - my $match = $opt{'api-info'} =~ m!^/(.*)/$! ? $1 : "^\Q$opt{'api-info'}\E\$"; - for $f (sort { lc $a cmp lc $b } keys %API) { - next unless $f =~ /$match/; - print "\n=== $f ===\n\n"; - my $info = 0; - if ($API{$f}{base} || $API{$f}{todo}) { - my $base = format_version($API{$f}{base} || $API{$f}{todo}); - print "Supported at least starting from perl-$base.\n"; - $info++; - } - if ($API{$f}{provided}) { - my $todo = $API{$f}{todo} ? format_version($API{$f}{todo}) : "5.003"; - print "Support by $ppport provided back to perl-$todo.\n"; - print "Support needs to be explicitly requested by NEED_$f.\n" if exists $need{$f}; - print "Depends on: ", join(', ', @{$depends{$f}}), ".\n" if exists $depends{$f}; - print "\n$hints{$f}" if exists $hints{$f}; - print "\nWARNING:\n$warnings{$f}" if exists $warnings{$f}; - $info++; - } - print "No portability information available.\n" unless $info; - $count++; - } - $count or print "Found no API matching '$opt{'api-info'}'."; - print "\n"; - exit 0; -} - -if (exists $opt{'list-provided'}) { - my $f; - for $f (sort { lc $a cmp lc $b } keys %API) { - next unless $API{$f}{provided}; - my @flags; - push @flags, 'explicit' if exists $need{$f}; - push @flags, 'depend' if exists $depends{$f}; - push @flags, 'hint' if exists $hints{$f}; - push @flags, 'warning' if exists $warnings{$f}; - my $flags = @flags ? ' ['.join(', ', @flags).']' : ''; - print "$f$flags\n"; - } - exit 0; -} - -my @files; -my @srcext = qw( .xs .c .h .cc .cpp -c.inc -xs.inc ); -my $srcext = join '|', map { quotemeta $_ } @srcext; - -if (@ARGV) { - my %seen; - for (@ARGV) { - if (-e) { - if (-f) { - push @files, $_ unless $seen{$_}++; - } - else { warn "'$_' is not a file.\n" } - } - else { - my @new = grep { -f } glob $_ - or warn "'$_' does not exist.\n"; - push @files, grep { !$seen{$_}++ } @new; - } - } -} -else { - eval { - require File::Find; - File::Find::find(sub { - $File::Find::name =~ /($srcext)$/i - and push @files, $File::Find::name; - }, '.'); - }; - if ($@) { - @files = map { glob "*$_" } @srcext; - } -} - -if (!@ARGV || $opt{filter}) { - my(@in, @out); - my %xsc = map { /(.*)\.xs$/ ? ("$1.c" => 1, "$1.cc" => 1) : () } @files; - for (@files) { - my $out = exists $xsc{$_} || /\b\Q$ppport\E$/i || !/($srcext)$/i; - push @{ $out ? \@out : \@in }, $_; - } - if (@ARGV && @out) { - warning("Skipping the following files (use --nofilter to avoid this):\n| ", join "\n| ", @out); - } - @files = @in; -} - -die "No input files given!\n" unless @files; - -my(%files, %global, %revreplace); -%revreplace = reverse %replace; -my $filename; -my $patch_opened = 0; - -for $filename (@files) { - unless (open IN, "<$filename") { - warn "Unable to read from $filename: $!\n"; - next; - } - - info("Scanning $filename ..."); - - my $c = do { local $/; }; - close IN; - - my %file = (orig => $c, changes => 0); - - # Temporarily remove C/XS comments and strings from the code - my @ccom; - - $c =~ s{ - ( ^$HS*\#$HS*include\b[^\r\n]+\b(?:\Q$ppport\E|XSUB\.h)\b[^\r\n]* - | ^$HS*\#$HS*(?:define|elif|if(?:def)?)\b[^\r\n]* ) - | ( ^$HS*\#[^\r\n]* - | "[^"\\]*(?:\\.[^"\\]*)*" - | '[^'\\]*(?:\\.[^'\\]*)*' - | / (?: \*[^*]*\*+(?:[^$ccs][^*]*\*+)* / | /[^\r\n]* ) ) - }{ defined $2 and push @ccom, $2; - defined $1 ? $1 : "$ccs$#ccom$cce" }mgsex; - - $file{ccom} = \@ccom; - $file{code} = $c; - $file{has_inc_ppport} = $c =~ /^$HS*#$HS*include[^\r\n]+\b\Q$ppport\E\b/m; - - my $func; - - for $func (keys %API) { - my $match = $func; - $match .= "|$revreplace{$func}" if exists $revreplace{$func}; - if ($c =~ /\b(?:Perl_)?($match)\b/) { - $file{uses_replace}{$1}++ if exists $revreplace{$func} && $1 eq $revreplace{$func}; - $file{uses_Perl}{$func}++ if $c =~ /\bPerl_$func\b/; - if (exists $API{$func}{provided}) { - $file{uses_provided}{$func}++; - if (!exists $API{$func}{base} || $API{$func}{base} > $opt{'compat-version'}) { - $file{uses}{$func}++; - my @deps = rec_depend($func); - if (@deps) { - $file{uses_deps}{$func} = \@deps; - for (@deps) { - $file{uses}{$_} = 0 unless exists $file{uses}{$_}; - } - } - for ($func, @deps) { - $file{needs}{$_} = 'static' if exists $need{$_}; - } - } - } - if (exists $API{$func}{todo} && $API{$func}{todo} > $opt{'compat-version'}) { - if ($c =~ /\b$func\b/) { - $file{uses_todo}{$func}++; - } - } - } - } - - while ($c =~ /^$HS*#$HS*define$HS+(NEED_(\w+?)(_GLOBAL)?)\b/mg) { - if (exists $need{$2}) { - $file{defined $3 ? 'needed_global' : 'needed_static'}{$2}++; - } - else { warning("Possibly wrong #define $1 in $filename") } - } - - for (qw(uses needs uses_todo needed_global needed_static)) { - for $func (keys %{$file{$_}}) { - push @{$global{$_}{$func}}, $filename; - } - } - - $files{$filename} = \%file; -} - -# Globally resolve NEED_'s -my $need; -for $need (keys %{$global{needs}}) { - if (@{$global{needs}{$need}} > 1) { - my @targets = @{$global{needs}{$need}}; - my @t = grep $files{$_}{needed_global}{$need}, @targets; - @targets = @t if @t; - @t = grep /\.xs$/i, @targets; - @targets = @t if @t; - my $target = shift @targets; - $files{$target}{needs}{$need} = 'global'; - for (@{$global{needs}{$need}}) { - $files{$_}{needs}{$need} = 'extern' if $_ ne $target; - } - } -} - -for $filename (@files) { - exists $files{$filename} or next; - - info("=== Analyzing $filename ==="); - - my %file = %{$files{$filename}}; - my $func; - my $c = $file{code}; - my $warnings = 0; - - for $func (sort keys %{$file{uses_Perl}}) { - if ($API{$func}{varargs}) { - unless ($API{$func}{nothxarg}) { - my $changes = ($c =~ s{\b(Perl_$func\s*\(\s*)(?!aTHX_?)(\)|[^\s)]*\))} - { $1 . ($2 eq ')' ? 'aTHX' : 'aTHX_ ') . $2 }ge); - if ($changes) { - warning("Doesn't pass interpreter argument aTHX to Perl_$func"); - $file{changes} += $changes; - } - } - } - else { - warning("Uses Perl_$func instead of $func"); - $file{changes} += ($c =~ s{\bPerl_$func(\s*)\((\s*aTHX_?)?\s*} - {$func$1(}g); - } - } - - for $func (sort keys %{$file{uses_replace}}) { - warning("Uses $func instead of $replace{$func}"); - $file{changes} += ($c =~ s/\b$func\b/$replace{$func}/g); - } - - for $func (sort keys %{$file{uses_provided}}) { - if ($file{uses}{$func}) { - if (exists $file{uses_deps}{$func}) { - diag("Uses $func, which depends on ", join(', ', @{$file{uses_deps}{$func}})); - } - else { - diag("Uses $func"); - } - } - $warnings += hint($func); - } - - unless ($opt{quiet}) { - for $func (sort keys %{$file{uses_todo}}) { - print "*** WARNING: Uses $func, which may not be portable below perl ", - format_version($API{$func}{todo}), ", even with '$ppport'\n"; - $warnings++; - } - } - - for $func (sort keys %{$file{needed_static}}) { - my $message = ''; - if (not exists $file{uses}{$func}) { - $message = "No need to define NEED_$func if $func is never used"; - } - elsif (exists $file{needs}{$func} && $file{needs}{$func} ne 'static') { - $message = "No need to define NEED_$func when already needed globally"; - } - if ($message) { - diag($message); - $file{changes} += ($c =~ s/^$HS*#$HS*define$HS+NEED_$func\b.*$LF//mg); - } - } - - for $func (sort keys %{$file{needed_global}}) { - my $message = ''; - if (not exists $global{uses}{$func}) { - $message = "No need to define NEED_${func}_GLOBAL if $func is never used"; - } - elsif (exists $file{needs}{$func}) { - if ($file{needs}{$func} eq 'extern') { - $message = "No need to define NEED_${func}_GLOBAL when already needed globally"; - } - elsif ($file{needs}{$func} eq 'static') { - $message = "No need to define NEED_${func}_GLOBAL when only used in this file"; - } - } - if ($message) { - diag($message); - $file{changes} += ($c =~ s/^$HS*#$HS*define$HS+NEED_${func}_GLOBAL\b.*$LF//mg); - } - } - - $file{needs_inc_ppport} = keys %{$file{uses}}; - - if ($file{needs_inc_ppport}) { - my $pp = ''; - - for $func (sort keys %{$file{needs}}) { - my $type = $file{needs}{$func}; - next if $type eq 'extern'; - my $suffix = $type eq 'global' ? '_GLOBAL' : ''; - unless (exists $file{"needed_$type"}{$func}) { - if ($type eq 'global') { - diag("Files [@{$global{needs}{$func}}] need $func, adding global request"); - } - else { - diag("File needs $func, adding static request"); - } - $pp .= "#define NEED_$func$suffix\n"; - } - } - - if ($pp && ($c =~ s/^(?=$HS*#$HS*define$HS+NEED_\w+)/$pp/m)) { - $pp = ''; - $file{changes}++; - } - - unless ($file{has_inc_ppport}) { - diag("Needs to include '$ppport'"); - $pp .= qq(#include "$ppport"\n) - } - - if ($pp) { - $file{changes} += ($c =~ s/^($HS*#$HS*define$HS+NEED_\w+.*?)^/$1$pp/ms) - || ($c =~ s/^(?=$HS*#$HS*include.*\Q$ppport\E)/$pp/m) - || ($c =~ s/^($HS*#$HS*include.*XSUB.*\s*?)^/$1$pp/m) - || ($c =~ s/^/$pp/); - } - } - else { - if ($file{has_inc_ppport}) { - diag("No need to include '$ppport'"); - $file{changes} += ($c =~ s/^$HS*?#$HS*include.*\Q$ppport\E.*?$LF//m); - } - } - - # put back in our C comments - my $ix; - my $cppc = 0; - my @ccom = @{$file{ccom}}; - for $ix (0 .. $#ccom) { - if (!$opt{cplusplus} && $ccom[$ix] =~ s!^//!!) { - $cppc++; - $file{changes} += $c =~ s/$rccs$ix$rcce/$ccs$ccom[$ix] $cce/; - } - else { - $c =~ s/$rccs$ix$rcce/$ccom[$ix]/; - } - } - - if ($cppc) { - my $s = $cppc != 1 ? 's' : ''; - warning("Uses $cppc C++ style comment$s, which is not portable"); - } - - my $s = $warnings != 1 ? 's' : ''; - my $warn = $warnings ? " ($warnings warning$s)" : ''; - info("Analysis completed$warn"); - - if ($file{changes}) { - if (exists $opt{copy}) { - my $newfile = "$filename$opt{copy}"; - if (-e $newfile) { - error("'$newfile' already exists, refusing to write copy of '$filename'"); - } - else { - local *F; - if (open F, ">$newfile") { - info("Writing copy of '$filename' with changes to '$newfile'"); - print F $c; - close F; - } - else { - error("Cannot open '$newfile' for writing: $!"); - } - } - } - elsif (exists $opt{patch} || $opt{changes}) { - if (exists $opt{patch}) { - unless ($patch_opened) { - if (open PATCH, ">$opt{patch}") { - $patch_opened = 1; - } - else { - error("Cannot open '$opt{patch}' for writing: $!"); - delete $opt{patch}; - $opt{changes} = 1; - goto fallback; - } - } - mydiff(\*PATCH, $filename, $c); - } - else { -fallback: - info("Suggested changes:"); - mydiff(\*STDOUT, $filename, $c); - } - } - else { - my $s = $file{changes} == 1 ? '' : 's'; - info("$file{changes} potentially required change$s detected"); - } - } - else { - info("Looks good"); - } -} - -close PATCH if $patch_opened; - -exit 0; - - -sub try_use { eval "use @_;"; return $@ eq '' } - -sub mydiff -{ - local *F = shift; - my($file, $str) = @_; - my $diff; - - if (exists $opt{diff}) { - $diff = run_diff($opt{diff}, $file, $str); - } - - if (!defined $diff and try_use('Text::Diff')) { - $diff = Text::Diff::diff($file, \$str, { STYLE => 'Unified' }); - $diff = <
$tmp") { - print F $str; - close F; - - if (open F, "$prog $file $tmp |") { - while () { - s/\Q$tmp\E/$file.patched/; - $diff .= $_; - } - close F; - unlink $tmp; - return $diff; - } - - unlink $tmp; - } - else { - error("Cannot open '$tmp' for writing: $!"); - } - - return undef; -} - -sub rec_depend -{ - my($func, $seen) = @_; - return () unless exists $depends{$func}; - $seen = {%{$seen||{}}}; - return () if $seen->{$func}++; - my %s; - grep !$s{$_}++, map { ($_, rec_depend($_, $seen)) } @{$depends{$func}}; -} - -sub parse_version -{ - my $ver = shift; - - if ($ver =~ /^(\d+)\.(\d+)\.(\d+)$/) { - return ($1, $2, $3); - } - elsif ($ver !~ /^\d+\.[\d_]+$/) { - die "cannot parse version '$ver'\n"; - } - - $ver =~ s/_//g; - $ver =~ s/$/000000/; - - my($r,$v,$s) = $ver =~ /(\d+)\.(\d{3})(\d{3})/; - - $v = int $v; - $s = int $s; - - if ($r < 5 || ($r == 5 && $v < 6)) { - if ($s % 10) { - die "cannot parse version '$ver'\n"; - } - } - - return ($r, $v, $s); -} - -sub format_version -{ - my $ver = shift; - - $ver =~ s/$/000000/; - my($r,$v,$s) = $ver =~ /(\d+)\.(\d{3})(\d{3})/; - - $v = int $v; - $s = int $s; - - if ($r < 5 || ($r == 5 && $v < 6)) { - if ($s % 10) { - die "invalid version '$ver'\n"; - } - $s /= 10; - - $ver = sprintf "%d.%03d", $r, $v; - $s > 0 and $ver .= sprintf "_%02d", $s; - - return $ver; - } - - return sprintf "%d.%d.%d", $r, $v, $s; -} - -sub info -{ - $opt{quiet} and return; - print @_, "\n"; -} - -sub diag -{ - $opt{quiet} and return; - $opt{diag} and print @_, "\n"; -} - -sub warning -{ - $opt{quiet} and return; - print "*** ", @_, "\n"; -} - -sub error -{ - print "*** ERROR: ", @_, "\n"; -} - -my %given_hints; -my %given_warnings; -sub hint -{ - $opt{quiet} and return; - my $func = shift; - my $rv = 0; - if (exists $warnings{$func} && !$given_warnings{$func}++) { - my $warn = $warnings{$func}; - $warn =~ s!^!*** !mg; - print "*** WARNING: $func\n", $warn; - $rv++; - } - if ($opt{hints} && exists $hints{$func} && !$given_hints{$func}++) { - my $hint = $hints{$func}; - $hint =~ s/^/ /mg; - print " --- hint for $func ---\n", $hint; - } - $rv; -} - -sub usage -{ - my($usage) = do { local(@ARGV,$/)=($0); <> } =~ /^=head\d$HS+SYNOPSIS\s*^(.*?)\s*^=/ms; - my %M = ( 'I' => '*' ); - $usage =~ s/^\s*perl\s+\S+/$^X $0/; - $usage =~ s/([A-Z])<([^>]+)>/$M{$1}$2$M{$1}/g; - - print < }; - my($copy) = $self =~ /^=head\d\s+COPYRIGHT\s*^(.*?)^=\w+/ms; - $copy =~ s/^(?=\S+)/ /gms; - $self =~ s/^$HS+Do NOT edit.*?(?=^-)/$copy/ms; - $self =~ s/^SKIP.*(?=^__DATA__)/SKIP -if (\@ARGV && \$ARGV[0] eq '--unstrip') { - eval { require Devel::PPPort }; - \$@ and die "Cannot require Devel::PPPort, please install.\\n"; - if (eval \$Devel::PPPort::VERSION < $VERSION) { - die "$0 was originally generated with Devel::PPPort $VERSION.\\n" - . "Your Devel::PPPort is only version \$Devel::PPPort::VERSION.\\n" - . "Please install a newer version, or --unstrip will not work.\\n"; - } - Devel::PPPort::WriteFile(\$0); - exit 0; -} -print <$0" or die "cannot strip $0: $!\n"; - print OUT "$pl$c\n"; - - exit 0; -} - -__DATA__ -*/ - -#ifndef _P_P_PORTABILITY_H_ -#define _P_P_PORTABILITY_H_ - -#ifndef DPPP_NAMESPACE -# define DPPP_NAMESPACE DPPP_ -#endif - -#define DPPP_CAT2(x,y) CAT2(x,y) -#define DPPP_(name) DPPP_CAT2(DPPP_NAMESPACE, name) - -#ifndef PERL_REVISION -# if !defined(__PATCHLEVEL_H_INCLUDED__) && !(defined(PATCHLEVEL) && defined(SUBVERSION)) -# define PERL_PATCHLEVEL_H_IMPLICIT -# include -# endif -# if !(defined(PERL_VERSION) || (defined(SUBVERSION) && defined(PATCHLEVEL))) -# include -# endif -# ifndef PERL_REVISION -# define PERL_REVISION (5) - /* Replace: 1 */ -# define PERL_VERSION PATCHLEVEL -# define PERL_SUBVERSION SUBVERSION - /* Replace PERL_PATCHLEVEL with PERL_VERSION */ - /* Replace: 0 */ -# endif -#endif - -#define _dpppDEC2BCD(dec) ((((dec)/100)<<8)|((((dec)%100)/10)<<4)|((dec)%10)) -#define PERL_BCDVERSION ((_dpppDEC2BCD(PERL_REVISION)<<24)|(_dpppDEC2BCD(PERL_VERSION)<<12)|_dpppDEC2BCD(PERL_SUBVERSION)) - -/* It is very unlikely that anyone will try to use this with Perl 6 - (or greater), but who knows. - */ -#if PERL_REVISION != 5 -# error ppport.h only works with Perl version 5 -#endif /* PERL_REVISION != 5 */ -#ifndef dTHR -# define dTHR dNOOP -#endif -#ifndef dTHX -# define dTHX dNOOP -#endif - -#ifndef dTHXa -# define dTHXa(x) dNOOP -#endif -#ifndef pTHX -# define pTHX void -#endif - -#ifndef pTHX_ -# define pTHX_ -#endif - -#ifndef aTHX -# define aTHX -#endif - -#ifndef aTHX_ -# define aTHX_ -#endif - -#if (PERL_BCDVERSION < 0x5006000) -# ifdef USE_THREADS -# define aTHXR thr -# define aTHXR_ thr, -# else -# define aTHXR -# define aTHXR_ -# endif -# define dTHXR dTHR -#else -# define aTHXR aTHX -# define aTHXR_ aTHX_ -# define dTHXR dTHX -#endif -#ifndef dTHXoa -# define dTHXoa(x) dTHXa(x) -#endif - -#ifdef I_LIMITS -# include -#endif - -#ifndef PERL_UCHAR_MIN -# define PERL_UCHAR_MIN ((unsigned char)0) -#endif - -#ifndef PERL_UCHAR_MAX -# ifdef UCHAR_MAX -# define PERL_UCHAR_MAX ((unsigned char)UCHAR_MAX) -# else -# ifdef MAXUCHAR -# define PERL_UCHAR_MAX ((unsigned char)MAXUCHAR) -# else -# define PERL_UCHAR_MAX ((unsigned char)~(unsigned)0) -# endif -# endif -#endif - -#ifndef PERL_USHORT_MIN -# define PERL_USHORT_MIN ((unsigned short)0) -#endif - -#ifndef PERL_USHORT_MAX -# ifdef USHORT_MAX -# define PERL_USHORT_MAX ((unsigned short)USHORT_MAX) -# else -# ifdef MAXUSHORT -# define PERL_USHORT_MAX ((unsigned short)MAXUSHORT) -# else -# ifdef USHRT_MAX -# define PERL_USHORT_MAX ((unsigned short)USHRT_MAX) -# else -# define PERL_USHORT_MAX ((unsigned short)~(unsigned)0) -# endif -# endif -# endif -#endif - -#ifndef PERL_SHORT_MAX -# ifdef SHORT_MAX -# define PERL_SHORT_MAX ((short)SHORT_MAX) -# else -# ifdef MAXSHORT /* Often used in */ -# define PERL_SHORT_MAX ((short)MAXSHORT) -# else -# ifdef SHRT_MAX -# define PERL_SHORT_MAX ((short)SHRT_MAX) -# else -# define PERL_SHORT_MAX ((short) (PERL_USHORT_MAX >> 1)) -# endif -# endif -# endif -#endif - -#ifndef PERL_SHORT_MIN -# ifdef SHORT_MIN -# define PERL_SHORT_MIN ((short)SHORT_MIN) -# else -# ifdef MINSHORT -# define PERL_SHORT_MIN ((short)MINSHORT) -# else -# ifdef SHRT_MIN -# define PERL_SHORT_MIN ((short)SHRT_MIN) -# else -# define PERL_SHORT_MIN (-PERL_SHORT_MAX - ((3 & -1) == 3)) -# endif -# endif -# endif -#endif - -#ifndef PERL_UINT_MAX -# ifdef UINT_MAX -# define PERL_UINT_MAX ((unsigned int)UINT_MAX) -# else -# ifdef MAXUINT -# define PERL_UINT_MAX ((unsigned int)MAXUINT) -# else -# define PERL_UINT_MAX (~(unsigned int)0) -# endif -# endif -#endif - -#ifndef PERL_UINT_MIN -# define PERL_UINT_MIN ((unsigned int)0) -#endif - -#ifndef PERL_INT_MAX -# ifdef INT_MAX -# define PERL_INT_MAX ((int)INT_MAX) -# else -# ifdef MAXINT /* Often used in */ -# define PERL_INT_MAX ((int)MAXINT) -# else -# define PERL_INT_MAX ((int)(PERL_UINT_MAX >> 1)) -# endif -# endif -#endif - -#ifndef PERL_INT_MIN -# ifdef INT_MIN -# define PERL_INT_MIN ((int)INT_MIN) -# else -# ifdef MININT -# define PERL_INT_MIN ((int)MININT) -# else -# define PERL_INT_MIN (-PERL_INT_MAX - ((3 & -1) == 3)) -# endif -# endif -#endif - -#ifndef PERL_ULONG_MAX -# ifdef ULONG_MAX -# define PERL_ULONG_MAX ((unsigned long)ULONG_MAX) -# else -# ifdef MAXULONG -# define PERL_ULONG_MAX ((unsigned long)MAXULONG) -# else -# define PERL_ULONG_MAX (~(unsigned long)0) -# endif -# endif -#endif - -#ifndef PERL_ULONG_MIN -# define PERL_ULONG_MIN ((unsigned long)0L) -#endif - -#ifndef PERL_LONG_MAX -# ifdef LONG_MAX -# define PERL_LONG_MAX ((long)LONG_MAX) -# else -# ifdef MAXLONG -# define PERL_LONG_MAX ((long)MAXLONG) -# else -# define PERL_LONG_MAX ((long) (PERL_ULONG_MAX >> 1)) -# endif -# endif -#endif - -#ifndef PERL_LONG_MIN -# ifdef LONG_MIN -# define PERL_LONG_MIN ((long)LONG_MIN) -# else -# ifdef MINLONG -# define PERL_LONG_MIN ((long)MINLONG) -# else -# define PERL_LONG_MIN (-PERL_LONG_MAX - ((3 & -1) == 3)) -# endif -# endif -#endif - -#if defined(HAS_QUAD) && (defined(convex) || defined(uts)) -# ifndef PERL_UQUAD_MAX -# ifdef ULONGLONG_MAX -# define PERL_UQUAD_MAX ((unsigned long long)ULONGLONG_MAX) -# else -# ifdef MAXULONGLONG -# define PERL_UQUAD_MAX ((unsigned long long)MAXULONGLONG) -# else -# define PERL_UQUAD_MAX (~(unsigned long long)0) -# endif -# endif -# endif - -# ifndef PERL_UQUAD_MIN -# define PERL_UQUAD_MIN ((unsigned long long)0L) -# endif - -# ifndef PERL_QUAD_MAX -# ifdef LONGLONG_MAX -# define PERL_QUAD_MAX ((long long)LONGLONG_MAX) -# else -# ifdef MAXLONGLONG -# define PERL_QUAD_MAX ((long long)MAXLONGLONG) -# else -# define PERL_QUAD_MAX ((long long) (PERL_UQUAD_MAX >> 1)) -# endif -# endif -# endif - -# ifndef PERL_QUAD_MIN -# ifdef LONGLONG_MIN -# define PERL_QUAD_MIN ((long long)LONGLONG_MIN) -# else -# ifdef MINLONGLONG -# define PERL_QUAD_MIN ((long long)MINLONGLONG) -# else -# define PERL_QUAD_MIN (-PERL_QUAD_MAX - ((3 & -1) == 3)) -# endif -# endif -# endif -#endif - -/* This is based on code from 5.003 perl.h */ -#ifdef HAS_QUAD -# ifdef cray -#ifndef IVTYPE -# define IVTYPE int -#endif - -#ifndef IV_MIN -# define IV_MIN PERL_INT_MIN -#endif - -#ifndef IV_MAX -# define IV_MAX PERL_INT_MAX -#endif - -#ifndef UV_MIN -# define UV_MIN PERL_UINT_MIN -#endif - -#ifndef UV_MAX -# define UV_MAX PERL_UINT_MAX -#endif - -# ifdef INTSIZE -#ifndef IVSIZE -# define IVSIZE INTSIZE -#endif - -# endif -# else -# if defined(convex) || defined(uts) -#ifndef IVTYPE -# define IVTYPE long long -#endif - -#ifndef IV_MIN -# define IV_MIN PERL_QUAD_MIN -#endif - -#ifndef IV_MAX -# define IV_MAX PERL_QUAD_MAX -#endif - -#ifndef UV_MIN -# define UV_MIN PERL_UQUAD_MIN -#endif - -#ifndef UV_MAX -# define UV_MAX PERL_UQUAD_MAX -#endif - -# ifdef LONGLONGSIZE -#ifndef IVSIZE -# define IVSIZE LONGLONGSIZE -#endif - -# endif -# else -#ifndef IVTYPE -# define IVTYPE long -#endif - -#ifndef IV_MIN -# define IV_MIN PERL_LONG_MIN -#endif - -#ifndef IV_MAX -# define IV_MAX PERL_LONG_MAX -#endif - -#ifndef UV_MIN -# define UV_MIN PERL_ULONG_MIN -#endif - -#ifndef UV_MAX -# define UV_MAX PERL_ULONG_MAX -#endif - -# ifdef LONGSIZE -#ifndef IVSIZE -# define IVSIZE LONGSIZE -#endif - -# endif -# endif -# endif -#ifndef IVSIZE -# define IVSIZE 8 -#endif - -#ifndef PERL_QUAD_MIN -# define PERL_QUAD_MIN IV_MIN -#endif - -#ifndef PERL_QUAD_MAX -# define PERL_QUAD_MAX IV_MAX -#endif - -#ifndef PERL_UQUAD_MIN -# define PERL_UQUAD_MIN UV_MIN -#endif - -#ifndef PERL_UQUAD_MAX -# define PERL_UQUAD_MAX UV_MAX -#endif - -#else -#ifndef IVTYPE -# define IVTYPE long -#endif - -#ifndef IV_MIN -# define IV_MIN PERL_LONG_MIN -#endif - -#ifndef IV_MAX -# define IV_MAX PERL_LONG_MAX -#endif - -#ifndef UV_MIN -# define UV_MIN PERL_ULONG_MIN -#endif - -#ifndef UV_MAX -# define UV_MAX PERL_ULONG_MAX -#endif - -#endif - -#ifndef IVSIZE -# ifdef LONGSIZE -# define IVSIZE LONGSIZE -# else -# define IVSIZE 4 /* A bold guess, but the best we can make. */ -# endif -#endif -#ifndef UVTYPE -# define UVTYPE unsigned IVTYPE -#endif - -#ifndef UVSIZE -# define UVSIZE IVSIZE -#endif -#ifndef sv_setuv -# define sv_setuv(sv, uv) \ - STMT_START { \ - UV TeMpUv = uv; \ - if (TeMpUv <= IV_MAX) \ - sv_setiv(sv, TeMpUv); \ - else \ - sv_setnv(sv, (double)TeMpUv); \ - } STMT_END -#endif -#ifndef newSVuv -# define newSVuv(uv) ((uv) <= IV_MAX ? newSViv((IV)uv) : newSVnv((NV)uv)) -#endif -#ifndef sv_2uv -# define sv_2uv(sv) ((PL_Sv = (sv)), (UV) (SvNOK(PL_Sv) ? SvNV(PL_Sv) : sv_2nv(PL_Sv))) -#endif - -#ifndef SvUVX -# define SvUVX(sv) ((UV)SvIVX(sv)) -#endif - -#ifndef SvUVXx -# define SvUVXx(sv) SvUVX(sv) -#endif - -#ifndef SvUV -# define SvUV(sv) (SvIOK(sv) ? SvUVX(sv) : sv_2uv(sv)) -#endif - -#ifndef SvUVx -# define SvUVx(sv) ((PL_Sv = (sv)), SvUV(PL_Sv)) -#endif - -/* Hint: sv_uv - * Always use the SvUVx() macro instead of sv_uv(). - */ -#ifndef sv_uv -# define sv_uv(sv) SvUVx(sv) -#endif - -#if !defined(SvUOK) && defined(SvIOK_UV) -# define SvUOK(sv) SvIOK_UV(sv) -#endif -#ifndef XST_mUV -# define XST_mUV(i,v) (ST(i) = sv_2mortal(newSVuv(v)) ) -#endif - -#ifndef XSRETURN_UV -# define XSRETURN_UV(v) STMT_START { XST_mUV(0,v); XSRETURN(1); } STMT_END -#endif -#ifndef PUSHu -# define PUSHu(u) STMT_START { sv_setuv(TARG, (UV)(u)); PUSHTARG; } STMT_END -#endif - -#ifndef XPUSHu -# define XPUSHu(u) STMT_START { sv_setuv(TARG, (UV)(u)); XPUSHTARG; } STMT_END -#endif - -#ifdef HAS_MEMCMP -#ifndef memNE -# define memNE(s1,s2,l) (memcmp(s1,s2,l)) -#endif - -#ifndef memEQ -# define memEQ(s1,s2,l) (!memcmp(s1,s2,l)) -#endif - -#else -#ifndef memNE -# define memNE(s1,s2,l) (bcmp(s1,s2,l)) -#endif - -#ifndef memEQ -# define memEQ(s1,s2,l) (!bcmp(s1,s2,l)) -#endif - -#endif -#ifndef MoveD -# define MoveD(s,d,n,t) memmove((char*)(d),(char*)(s), (n) * sizeof(t)) -#endif - -#ifndef CopyD -# define CopyD(s,d,n,t) memcpy((char*)(d),(char*)(s), (n) * sizeof(t)) -#endif - -#ifdef HAS_MEMSET -#ifndef ZeroD -# define ZeroD(d,n,t) memzero((char*)(d), (n) * sizeof(t)) -#endif - -#else -#ifndef ZeroD -# define ZeroD(d,n,t) ((void)memzero((char*)(d), (n) * sizeof(t)), d) -#endif - -#endif -#ifndef PoisonWith -# define PoisonWith(d,n,t,b) (void)memset((char*)(d), (U8)(b), (n) * sizeof(t)) -#endif - -#ifndef PoisonNew -# define PoisonNew(d,n,t) PoisonWith(d,n,t,0xAB) -#endif - -#ifndef PoisonFree -# define PoisonFree(d,n,t) PoisonWith(d,n,t,0xEF) -#endif - -#ifndef Poison -# define Poison(d,n,t) PoisonFree(d,n,t) -#endif -#ifndef Newx -# define Newx(v,n,t) New(0,v,n,t) -#endif - -#ifndef Newxc -# define Newxc(v,n,t,c) Newc(0,v,n,t,c) -#endif - -#ifndef Newxz -# define Newxz(v,n,t) Newz(0,v,n,t) -#endif - -#ifndef PERL_UNUSED_DECL -# ifdef HASATTRIBUTE -# if (defined(__GNUC__) && defined(__cplusplus)) || defined(__INTEL_COMPILER) -# define PERL_UNUSED_DECL -# else -# define PERL_UNUSED_DECL __attribute__((unused)) -# endif -# else -# define PERL_UNUSED_DECL -# endif -#endif - -#ifndef PERL_UNUSED_ARG -# if defined(lint) && defined(S_SPLINT_S) /* www.splint.org */ -# include -# define PERL_UNUSED_ARG(x) NOTE(ARGUNUSED(x)) -# else -# define PERL_UNUSED_ARG(x) ((void)x) -# endif -#endif - -#ifndef PERL_UNUSED_VAR -# define PERL_UNUSED_VAR(x) ((void)x) -#endif - -#ifndef PERL_UNUSED_CONTEXT -# ifdef USE_ITHREADS -# define PERL_UNUSED_CONTEXT PERL_UNUSED_ARG(my_perl) -# else -# define PERL_UNUSED_CONTEXT -# endif -#endif -#ifndef NOOP -# define NOOP /*EMPTY*/(void)0 -#endif - -#ifndef dNOOP -# define dNOOP extern int /*@unused@*/ Perl___notused PERL_UNUSED_DECL -#endif - -#ifndef NVTYPE -# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) -# define NVTYPE long double -# else -# define NVTYPE double -# endif -typedef NVTYPE NV; -#endif - -#ifndef INT2PTR -# if (IVSIZE == PTRSIZE) && (UVSIZE == PTRSIZE) -# define PTRV UV -# define INT2PTR(any,d) (any)(d) -# else -# if PTRSIZE == LONGSIZE -# define PTRV unsigned long -# else -# define PTRV unsigned -# endif -# define INT2PTR(any,d) (any)(PTRV)(d) -# endif -#endif - -#ifndef PTR2ul -# if PTRSIZE == LONGSIZE -# define PTR2ul(p) (unsigned long)(p) -# else -# define PTR2ul(p) INT2PTR(unsigned long,p) -# endif -#endif -#ifndef PTR2nat -# define PTR2nat(p) (PTRV)(p) -#endif - -#ifndef NUM2PTR -# define NUM2PTR(any,d) (any)PTR2nat(d) -#endif - -#ifndef PTR2IV -# define PTR2IV(p) INT2PTR(IV,p) -#endif - -#ifndef PTR2UV -# define PTR2UV(p) INT2PTR(UV,p) -#endif - -#ifndef PTR2NV -# define PTR2NV(p) NUM2PTR(NV,p) -#endif - -#undef START_EXTERN_C -#undef END_EXTERN_C -#undef EXTERN_C -#ifdef __cplusplus -# define START_EXTERN_C extern "C" { -# define END_EXTERN_C } -# define EXTERN_C extern "C" -#else -# define START_EXTERN_C -# define END_EXTERN_C -# define EXTERN_C extern -#endif - -#if defined(PERL_GCC_PEDANTIC) -# ifndef PERL_GCC_BRACE_GROUPS_FORBIDDEN -# define PERL_GCC_BRACE_GROUPS_FORBIDDEN -# endif -#endif - -#if defined(__GNUC__) && !defined(PERL_GCC_BRACE_GROUPS_FORBIDDEN) && !defined(__cplusplus) -# ifndef PERL_USE_GCC_BRACE_GROUPS -# define PERL_USE_GCC_BRACE_GROUPS -# endif -#endif - -#undef STMT_START -#undef STMT_END -#ifdef PERL_USE_GCC_BRACE_GROUPS -# define STMT_START (void)( /* gcc supports ``({ STATEMENTS; })'' */ -# define STMT_END ) -#else -# if defined(VOIDFLAGS) && (VOIDFLAGS) && (defined(sun) || defined(__sun__)) && !defined(__GNUC__) -# define STMT_START if (1) -# define STMT_END else (void)0 -# else -# define STMT_START do -# define STMT_END while (0) -# endif -#endif -#ifndef boolSV -# define boolSV(b) ((b) ? &PL_sv_yes : &PL_sv_no) -#endif - -/* DEFSV appears first in 5.004_56 */ -#ifndef DEFSV -# define DEFSV GvSV(PL_defgv) -#endif - -#ifndef SAVE_DEFSV -# define SAVE_DEFSV SAVESPTR(GvSV(PL_defgv)) -#endif - -#ifndef DEFSV_set -# define DEFSV_set(sv) (DEFSV = (sv)) -#endif - -/* Older perls (<=5.003) lack AvFILLp */ -#ifndef AvFILLp -# define AvFILLp AvFILL -#endif -#ifndef ERRSV -# define ERRSV get_sv("@",FALSE) -#endif - -/* Hint: gv_stashpvn - * This function's backport doesn't support the length parameter, but - * rather ignores it. Portability can only be ensured if the length - * parameter is used for speed reasons, but the length can always be - * correctly computed from the string argument. - */ -#ifndef gv_stashpvn -# define gv_stashpvn(str,len,create) gv_stashpv(str,create) -#endif - -/* Replace: 1 */ -#ifndef get_cv -# define get_cv perl_get_cv -#endif - -#ifndef get_sv -# define get_sv perl_get_sv -#endif - -#ifndef get_av -# define get_av perl_get_av -#endif - -#ifndef get_hv -# define get_hv perl_get_hv -#endif - -/* Replace: 0 */ -#ifndef dUNDERBAR -# define dUNDERBAR dNOOP -#endif - -#ifndef UNDERBAR -# define UNDERBAR DEFSV -#endif -#ifndef dAX -# define dAX I32 ax = MARK - PL_stack_base + 1 -#endif - -#ifndef dITEMS -# define dITEMS I32 items = SP - MARK -#endif -#ifndef dXSTARG -# define dXSTARG SV * targ = sv_newmortal() -#endif -#ifndef dAXMARK -# define dAXMARK I32 ax = POPMARK; \ - register SV ** const mark = PL_stack_base + ax++ -#endif -#ifndef XSprePUSH -# define XSprePUSH (sp = PL_stack_base + ax - 1) -#endif - -#if (PERL_BCDVERSION < 0x5005000) -# undef XSRETURN -# define XSRETURN(off) \ - STMT_START { \ - PL_stack_sp = PL_stack_base + ax + ((off) - 1); \ - return; \ - } STMT_END -#endif -#ifndef XSPROTO -# define XSPROTO(name) void name(pTHX_ CV* cv) -#endif - -#ifndef SVfARG -# define SVfARG(p) ((void*)(p)) -#endif -#ifndef PERL_ABS -# define PERL_ABS(x) ((x) < 0 ? -(x) : (x)) -#endif -#ifndef dVAR -# define dVAR dNOOP -#endif -#ifndef SVf -# define SVf "_" -#endif -#ifndef UTF8_MAXBYTES -# define UTF8_MAXBYTES UTF8_MAXLEN -#endif -#ifndef CPERLscope -# define CPERLscope(x) x -#endif -#ifndef PERL_HASH -# define PERL_HASH(hash,str,len) \ - STMT_START { \ - const char *s_PeRlHaSh = str; \ - I32 i_PeRlHaSh = len; \ - U32 hash_PeRlHaSh = 0; \ - while (i_PeRlHaSh--) \ - hash_PeRlHaSh = hash_PeRlHaSh * 33 + *s_PeRlHaSh++; \ - (hash) = hash_PeRlHaSh; \ - } STMT_END -#endif - -#ifndef PERLIO_FUNCS_DECL -# ifdef PERLIO_FUNCS_CONST -# define PERLIO_FUNCS_DECL(funcs) const PerlIO_funcs funcs -# define PERLIO_FUNCS_CAST(funcs) (PerlIO_funcs*)(funcs) -# else -# define PERLIO_FUNCS_DECL(funcs) PerlIO_funcs funcs -# define PERLIO_FUNCS_CAST(funcs) (funcs) -# endif -#endif - -/* provide these typedefs for older perls */ -#if (PERL_BCDVERSION < 0x5009003) - -# ifdef ARGSproto -typedef OP* (CPERLscope(*Perl_ppaddr_t))(ARGSproto); -# else -typedef OP* (CPERLscope(*Perl_ppaddr_t))(pTHX); -# endif - -typedef OP* (CPERLscope(*Perl_check_t)) (pTHX_ OP*); - -#endif -#ifndef isPSXSPC -# define isPSXSPC(c) (isSPACE(c) || (c) == '\v') -#endif - -#ifndef isBLANK -# define isBLANK(c) ((c) == ' ' || (c) == '\t') -#endif - -#ifdef EBCDIC -#ifndef isALNUMC -# define isALNUMC(c) isalnum(c) -#endif - -#ifndef isASCII -# define isASCII(c) isascii(c) -#endif - -#ifndef isCNTRL -# define isCNTRL(c) iscntrl(c) -#endif - -#ifndef isGRAPH -# define isGRAPH(c) isgraph(c) -#endif - -#ifndef isPRINT -# define isPRINT(c) isprint(c) -#endif - -#ifndef isPUNCT -# define isPUNCT(c) ispunct(c) -#endif - -#ifndef isXDIGIT -# define isXDIGIT(c) isxdigit(c) -#endif - -#else -# if (PERL_BCDVERSION < 0x5010000) -/* Hint: isPRINT - * The implementation in older perl versions includes all of the - * isSPACE() characters, which is wrong. The version provided by - * Devel::PPPort always overrides a present buggy version. - */ -# undef isPRINT -# endif -#ifndef isALNUMC -# define isALNUMC(c) (isALPHA(c) || isDIGIT(c)) -#endif - -#ifndef isASCII -# define isASCII(c) ((c) <= 127) -#endif - -#ifndef isCNTRL -# define isCNTRL(c) ((c) < ' ' || (c) == 127) -#endif - -#ifndef isGRAPH -# define isGRAPH(c) (isALNUM(c) || isPUNCT(c)) -#endif - -#ifndef isPRINT -# define isPRINT(c) (((c) >= 32 && (c) < 127)) -#endif - -#ifndef isPUNCT -# define isPUNCT(c) (((c) >= 33 && (c) <= 47) || ((c) >= 58 && (c) <= 64) || ((c) >= 91 && (c) <= 96) || ((c) >= 123 && (c) <= 126)) -#endif - -#ifndef isXDIGIT -# define isXDIGIT(c) (isDIGIT(c) || ((c) >= 'a' && (c) <= 'f') || ((c) >= 'A' && (c) <= 'F')) -#endif - -#endif - -#ifndef PERL_SIGNALS_UNSAFE_FLAG - -#define PERL_SIGNALS_UNSAFE_FLAG 0x0001 - -#if (PERL_BCDVERSION < 0x5008000) -# define D_PPP_PERL_SIGNALS_INIT PERL_SIGNALS_UNSAFE_FLAG -#else -# define D_PPP_PERL_SIGNALS_INIT 0 -#endif - -#if defined(NEED_PL_signals) -static U32 DPPP_(my_PL_signals) = D_PPP_PERL_SIGNALS_INIT; -#elif defined(NEED_PL_signals_GLOBAL) -U32 DPPP_(my_PL_signals) = D_PPP_PERL_SIGNALS_INIT; -#else -extern U32 DPPP_(my_PL_signals); -#endif -#define PL_signals DPPP_(my_PL_signals) - -#endif - -/* Hint: PL_ppaddr - * Calling an op via PL_ppaddr requires passing a context argument - * for threaded builds. Since the context argument is different for - * 5.005 perls, you can use aTHXR (supplied by ppport.h), which will - * automatically be defined as the correct argument. - */ - -#if (PERL_BCDVERSION <= 0x5005005) -/* Replace: 1 */ -# define PL_ppaddr ppaddr -# define PL_no_modify no_modify -/* Replace: 0 */ -#endif - -#if (PERL_BCDVERSION <= 0x5004005) -/* Replace: 1 */ -# define PL_DBsignal DBsignal -# define PL_DBsingle DBsingle -# define PL_DBsub DBsub -# define PL_DBtrace DBtrace -# define PL_Sv Sv -# define PL_bufend bufend -# define PL_bufptr bufptr -# define PL_compiling compiling -# define PL_copline copline -# define PL_curcop curcop -# define PL_curstash curstash -# define PL_debstash debstash -# define PL_defgv defgv -# define PL_diehook diehook -# define PL_dirty dirty -# define PL_dowarn dowarn -# define PL_errgv errgv -# define PL_error_count error_count -# define PL_expect expect -# define PL_hexdigit hexdigit -# define PL_hints hints -# define PL_in_my in_my -# define PL_laststatval laststatval -# define PL_lex_state lex_state -# define PL_lex_stuff lex_stuff -# define PL_linestr linestr -# define PL_na na -# define PL_perl_destruct_level perl_destruct_level -# define PL_perldb perldb -# define PL_rsfp_filters rsfp_filters -# define PL_rsfp rsfp -# define PL_stack_base stack_base -# define PL_stack_sp stack_sp -# define PL_statcache statcache -# define PL_stdingv stdingv -# define PL_sv_arenaroot sv_arenaroot -# define PL_sv_no sv_no -# define PL_sv_undef sv_undef -# define PL_sv_yes sv_yes -# define PL_tainted tainted -# define PL_tainting tainting -# define PL_tokenbuf tokenbuf -/* Replace: 0 */ -#endif - -/* Warning: PL_parser - * For perl versions earlier than 5.9.5, this is an always - * non-NULL dummy. Also, it cannot be dereferenced. Don't - * use it if you can avoid is and unless you absolutely know - * what you're doing. - * If you always check that PL_parser is non-NULL, you can - * define DPPP_PL_parser_NO_DUMMY to avoid the creation of - * a dummy parser structure. - */ - -#if (PERL_BCDVERSION >= 0x5009005) -# ifdef DPPP_PL_parser_NO_DUMMY -# define D_PPP_my_PL_parser_var(var) ((PL_parser ? PL_parser : \ - (croak("panic: PL_parser == NULL in %s:%d", \ - __FILE__, __LINE__), (yy_parser *) NULL))->var) -# else -# ifdef DPPP_PL_parser_NO_DUMMY_WARNING -# define D_PPP_parser_dummy_warning(var) -# else -# define D_PPP_parser_dummy_warning(var) \ - warn("warning: dummy PL_" #var " used in %s:%d", __FILE__, __LINE__), -# endif -# define D_PPP_my_PL_parser_var(var) ((PL_parser ? PL_parser : \ - (D_PPP_parser_dummy_warning(var) &DPPP_(dummy_PL_parser)))->var) -#if defined(NEED_PL_parser) -static yy_parser DPPP_(dummy_PL_parser); -#elif defined(NEED_PL_parser_GLOBAL) -yy_parser DPPP_(dummy_PL_parser); -#else -extern yy_parser DPPP_(dummy_PL_parser); -#endif - -# endif - -/* PL_expect, PL_copline, PL_rsfp, PL_rsfp_filters, PL_linestr, PL_bufptr, PL_bufend, PL_lex_state, PL_lex_stuff, PL_tokenbuf depends on PL_parser */ -/* Warning: PL_expect, PL_copline, PL_rsfp, PL_rsfp_filters, PL_linestr, PL_bufptr, PL_bufend, PL_lex_state, PL_lex_stuff, PL_tokenbuf - * Do not use this variable unless you know exactly what you're - * doint. It is internal to the perl parser and may change or even - * be removed in the future. As of perl 5.9.5, you have to check - * for (PL_parser != NULL) for this variable to have any effect. - * An always non-NULL PL_parser dummy is provided for earlier - * perl versions. - * If PL_parser is NULL when you try to access this variable, a - * dummy is being accessed instead and a warning is issued unless - * you define DPPP_PL_parser_NO_DUMMY_WARNING. - * If DPPP_PL_parser_NO_DUMMY is defined, the code trying to access - * this variable will croak with a panic message. - */ - -# define PL_expect D_PPP_my_PL_parser_var(expect) -# define PL_copline D_PPP_my_PL_parser_var(copline) -# define PL_rsfp D_PPP_my_PL_parser_var(rsfp) -# define PL_rsfp_filters D_PPP_my_PL_parser_var(rsfp_filters) -# define PL_linestr D_PPP_my_PL_parser_var(linestr) -# define PL_bufptr D_PPP_my_PL_parser_var(bufptr) -# define PL_bufend D_PPP_my_PL_parser_var(bufend) -# define PL_lex_state D_PPP_my_PL_parser_var(lex_state) -# define PL_lex_stuff D_PPP_my_PL_parser_var(lex_stuff) -# define PL_tokenbuf D_PPP_my_PL_parser_var(tokenbuf) -# define PL_in_my D_PPP_my_PL_parser_var(in_my) -# define PL_in_my_stash D_PPP_my_PL_parser_var(in_my_stash) -# define PL_error_count D_PPP_my_PL_parser_var(error_count) - - -#else - -/* ensure that PL_parser != NULL and cannot be dereferenced */ -# define PL_parser ((void *) 1) - -#endif -#ifndef mPUSHs -# define mPUSHs(s) PUSHs(sv_2mortal(s)) -#endif - -#ifndef PUSHmortal -# define PUSHmortal PUSHs(sv_newmortal()) -#endif - -#ifndef mPUSHp -# define mPUSHp(p,l) sv_setpvn(PUSHmortal, (p), (l)) -#endif - -#ifndef mPUSHn -# define mPUSHn(n) sv_setnv(PUSHmortal, (NV)(n)) -#endif - -#ifndef mPUSHi -# define mPUSHi(i) sv_setiv(PUSHmortal, (IV)(i)) -#endif - -#ifndef mPUSHu -# define mPUSHu(u) sv_setuv(PUSHmortal, (UV)(u)) -#endif -#ifndef mXPUSHs -# define mXPUSHs(s) XPUSHs(sv_2mortal(s)) -#endif - -#ifndef XPUSHmortal -# define XPUSHmortal XPUSHs(sv_newmortal()) -#endif - -#ifndef mXPUSHp -# define mXPUSHp(p,l) STMT_START { EXTEND(sp,1); sv_setpvn(PUSHmortal, (p), (l)); } STMT_END -#endif - -#ifndef mXPUSHn -# define mXPUSHn(n) STMT_START { EXTEND(sp,1); sv_setnv(PUSHmortal, (NV)(n)); } STMT_END -#endif - -#ifndef mXPUSHi -# define mXPUSHi(i) STMT_START { EXTEND(sp,1); sv_setiv(PUSHmortal, (IV)(i)); } STMT_END -#endif - -#ifndef mXPUSHu -# define mXPUSHu(u) STMT_START { EXTEND(sp,1); sv_setuv(PUSHmortal, (UV)(u)); } STMT_END -#endif - -/* Replace: 1 */ -#ifndef call_sv -# define call_sv perl_call_sv -#endif - -#ifndef call_pv -# define call_pv perl_call_pv -#endif - -#ifndef call_argv -# define call_argv perl_call_argv -#endif - -#ifndef call_method -# define call_method perl_call_method -#endif -#ifndef eval_sv -# define eval_sv perl_eval_sv -#endif - -/* Replace: 0 */ -#ifndef PERL_LOADMOD_DENY -# define PERL_LOADMOD_DENY 0x1 -#endif - -#ifndef PERL_LOADMOD_NOIMPORT -# define PERL_LOADMOD_NOIMPORT 0x2 -#endif - -#ifndef PERL_LOADMOD_IMPORT_OPS -# define PERL_LOADMOD_IMPORT_OPS 0x4 -#endif - -#ifndef G_METHOD -# define G_METHOD 64 -# ifdef call_sv -# undef call_sv -# endif -# if (PERL_BCDVERSION < 0x5006000) -# define call_sv(sv, flags) ((flags) & G_METHOD ? perl_call_method((char *) SvPV_nolen_const(sv), \ - (flags) & ~G_METHOD) : perl_call_sv(sv, flags)) -# else -# define call_sv(sv, flags) ((flags) & G_METHOD ? Perl_call_method(aTHX_ (char *) SvPV_nolen_const(sv), \ - (flags) & ~G_METHOD) : Perl_call_sv(aTHX_ sv, flags)) -# endif -#endif - -/* Replace perl_eval_pv with eval_pv */ - -#ifndef eval_pv -#if defined(NEED_eval_pv) -static SV* DPPP_(my_eval_pv)(char *p, I32 croak_on_error); -static -#else -extern SV* DPPP_(my_eval_pv)(char *p, I32 croak_on_error); -#endif - -#ifdef eval_pv -# undef eval_pv -#endif -#define eval_pv(a,b) DPPP_(my_eval_pv)(aTHX_ a,b) -#define Perl_eval_pv DPPP_(my_eval_pv) - -#if defined(NEED_eval_pv) || defined(NEED_eval_pv_GLOBAL) - -SV* -DPPP_(my_eval_pv)(char *p, I32 croak_on_error) -{ - dSP; - SV* sv = newSVpv(p, 0); - - PUSHMARK(sp); - eval_sv(sv, G_SCALAR); - SvREFCNT_dec(sv); - - SPAGAIN; - sv = POPs; - PUTBACK; - - if (croak_on_error && SvTRUE(GvSV(errgv))) - croak(SvPVx(GvSV(errgv), na)); - - return sv; -} - -#endif -#endif - -#ifndef vload_module -#if defined(NEED_vload_module) -static void DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args); -static -#else -extern void DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args); -#endif - -#ifdef vload_module -# undef vload_module -#endif -#define vload_module(a,b,c,d) DPPP_(my_vload_module)(aTHX_ a,b,c,d) -#define Perl_vload_module DPPP_(my_vload_module) - -#if defined(NEED_vload_module) || defined(NEED_vload_module_GLOBAL) - -void -DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args) -{ - dTHR; - dVAR; - OP *veop, *imop; - - OP * const modname = newSVOP(OP_CONST, 0, name); - /* 5.005 has a somewhat hacky force_normal that doesn't croak on - SvREADONLY() if PL_compling is true. Current perls take care in - ck_require() to correctly turn off SvREADONLY before calling - force_normal_flags(). This seems a better fix than fudging PL_compling - */ - SvREADONLY_off(((SVOP*)modname)->op_sv); - modname->op_private |= OPpCONST_BARE; - if (ver) { - veop = newSVOP(OP_CONST, 0, ver); - } - else - veop = NULL; - if (flags & PERL_LOADMOD_NOIMPORT) { - imop = sawparens(newNULLLIST()); - } - else if (flags & PERL_LOADMOD_IMPORT_OPS) { - imop = va_arg(*args, OP*); - } - else { - SV *sv; - imop = NULL; - sv = va_arg(*args, SV*); - while (sv) { - imop = append_elem(OP_LIST, imop, newSVOP(OP_CONST, 0, sv)); - sv = va_arg(*args, SV*); - } - } - { - const line_t ocopline = PL_copline; - COP * const ocurcop = PL_curcop; - const int oexpect = PL_expect; - -#if (PERL_BCDVERSION >= 0x5004000) - utilize(!(flags & PERL_LOADMOD_DENY), start_subparse(FALSE, 0), - veop, modname, imop); -#else - utilize(!(flags & PERL_LOADMOD_DENY), start_subparse(), - modname, imop); -#endif - PL_expect = oexpect; - PL_copline = ocopline; - PL_curcop = ocurcop; - } -} - -#endif -#endif - -#ifndef load_module -#if defined(NEED_load_module) -static void DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...); -static -#else -extern void DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...); -#endif - -#ifdef load_module -# undef load_module -#endif -#define load_module DPPP_(my_load_module) -#define Perl_load_module DPPP_(my_load_module) - -#if defined(NEED_load_module) || defined(NEED_load_module_GLOBAL) - -void -DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...) -{ - va_list args; - va_start(args, ver); - vload_module(flags, name, ver, &args); - va_end(args); -} - -#endif -#endif -#ifndef newRV_inc -# define newRV_inc(sv) newRV(sv) /* Replace */ -#endif - -#ifndef newRV_noinc -#if defined(NEED_newRV_noinc) -static SV * DPPP_(my_newRV_noinc)(SV *sv); -static -#else -extern SV * DPPP_(my_newRV_noinc)(SV *sv); -#endif - -#ifdef newRV_noinc -# undef newRV_noinc -#endif -#define newRV_noinc(a) DPPP_(my_newRV_noinc)(aTHX_ a) -#define Perl_newRV_noinc DPPP_(my_newRV_noinc) - -#if defined(NEED_newRV_noinc) || defined(NEED_newRV_noinc_GLOBAL) -SV * -DPPP_(my_newRV_noinc)(SV *sv) -{ - SV *rv = (SV *)newRV(sv); - SvREFCNT_dec(sv); - return rv; -} -#endif -#endif - -/* Hint: newCONSTSUB - * Returns a CV* as of perl-5.7.1. This return value is not supported - * by Devel::PPPort. - */ - -/* newCONSTSUB from IO.xs is in the core starting with 5.004_63 */ -#if (PERL_BCDVERSION < 0x5004063) && (PERL_BCDVERSION != 0x5004005) -#if defined(NEED_newCONSTSUB) -static void DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv); -static -#else -extern void DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv); -#endif - -#ifdef newCONSTSUB -# undef newCONSTSUB -#endif -#define newCONSTSUB(a,b,c) DPPP_(my_newCONSTSUB)(aTHX_ a,b,c) -#define Perl_newCONSTSUB DPPP_(my_newCONSTSUB) - -#if defined(NEED_newCONSTSUB) || defined(NEED_newCONSTSUB_GLOBAL) - -/* This is just a trick to avoid a dependency of newCONSTSUB on PL_parser */ -/* (There's no PL_parser in perl < 5.005, so this is completely safe) */ -#define D_PPP_PL_copline PL_copline - -void -DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv) -{ - U32 oldhints = PL_hints; - HV *old_cop_stash = PL_curcop->cop_stash; - HV *old_curstash = PL_curstash; - line_t oldline = PL_curcop->cop_line; - PL_curcop->cop_line = D_PPP_PL_copline; - - PL_hints &= ~HINT_BLOCK_SCOPE; - if (stash) - PL_curstash = PL_curcop->cop_stash = stash; - - newSUB( - -#if (PERL_BCDVERSION < 0x5003022) - start_subparse(), -#elif (PERL_BCDVERSION == 0x5003022) - start_subparse(0), -#else /* 5.003_23 onwards */ - start_subparse(FALSE, 0), -#endif - - newSVOP(OP_CONST, 0, newSVpv((char *) name, 0)), - newSVOP(OP_CONST, 0, &PL_sv_no), /* SvPV(&PL_sv_no) == "" -- GMB */ - newSTATEOP(0, Nullch, newSVOP(OP_CONST, 0, sv)) - ); - - PL_hints = oldhints; - PL_curcop->cop_stash = old_cop_stash; - PL_curstash = old_curstash; - PL_curcop->cop_line = oldline; -} -#endif -#endif - -/* - * Boilerplate macros for initializing and accessing interpreter-local - * data from C. All statics in extensions should be reworked to use - * this, if you want to make the extension thread-safe. See ext/re/re.xs - * for an example of the use of these macros. - * - * Code that uses these macros is responsible for the following: - * 1. #define MY_CXT_KEY to a unique string, e.g. "DynaLoader_guts" - * 2. Declare a typedef named my_cxt_t that is a structure that contains - * all the data that needs to be interpreter-local. - * 3. Use the START_MY_CXT macro after the declaration of my_cxt_t. - * 4. Use the MY_CXT_INIT macro such that it is called exactly once - * (typically put in the BOOT: section). - * 5. Use the members of the my_cxt_t structure everywhere as - * MY_CXT.member. - * 6. Use the dMY_CXT macro (a declaration) in all the functions that - * access MY_CXT. - */ - -#if defined(MULTIPLICITY) || defined(PERL_OBJECT) || \ - defined(PERL_CAPI) || defined(PERL_IMPLICIT_CONTEXT) - -#ifndef START_MY_CXT - -/* This must appear in all extensions that define a my_cxt_t structure, - * right after the definition (i.e. at file scope). The non-threads - * case below uses it to declare the data as static. */ -#define START_MY_CXT - -#if (PERL_BCDVERSION < 0x5004068) -/* Fetches the SV that keeps the per-interpreter data. */ -#define dMY_CXT_SV \ - SV *my_cxt_sv = get_sv(MY_CXT_KEY, FALSE) -#else /* >= perl5.004_68 */ -#define dMY_CXT_SV \ - SV *my_cxt_sv = *hv_fetch(PL_modglobal, MY_CXT_KEY, \ - sizeof(MY_CXT_KEY)-1, TRUE) -#endif /* < perl5.004_68 */ - -/* This declaration should be used within all functions that use the - * interpreter-local data. */ -#define dMY_CXT \ - dMY_CXT_SV; \ - my_cxt_t *my_cxtp = INT2PTR(my_cxt_t*,SvUV(my_cxt_sv)) - -/* Creates and zeroes the per-interpreter data. - * (We allocate my_cxtp in a Perl SV so that it will be released when - * the interpreter goes away.) */ -#define MY_CXT_INIT \ - dMY_CXT_SV; \ - /* newSV() allocates one more than needed */ \ - my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ - Zero(my_cxtp, 1, my_cxt_t); \ - sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) - -/* This macro must be used to access members of the my_cxt_t structure. - * e.g. MYCXT.some_data */ -#define MY_CXT (*my_cxtp) - -/* Judicious use of these macros can reduce the number of times dMY_CXT - * is used. Use is similar to pTHX, aTHX etc. */ -#define pMY_CXT my_cxt_t *my_cxtp -#define pMY_CXT_ pMY_CXT, -#define _pMY_CXT ,pMY_CXT -#define aMY_CXT my_cxtp -#define aMY_CXT_ aMY_CXT, -#define _aMY_CXT ,aMY_CXT - -#endif /* START_MY_CXT */ - -#ifndef MY_CXT_CLONE -/* Clones the per-interpreter data. */ -#define MY_CXT_CLONE \ - dMY_CXT_SV; \ - my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ - Copy(INT2PTR(my_cxt_t*, SvUV(my_cxt_sv)), my_cxtp, 1, my_cxt_t);\ - sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) -#endif - -#else /* single interpreter */ - -#ifndef START_MY_CXT - -#define START_MY_CXT static my_cxt_t my_cxt; -#define dMY_CXT_SV dNOOP -#define dMY_CXT dNOOP -#define MY_CXT_INIT NOOP -#define MY_CXT my_cxt - -#define pMY_CXT void -#define pMY_CXT_ -#define _pMY_CXT -#define aMY_CXT -#define aMY_CXT_ -#define _aMY_CXT - -#endif /* START_MY_CXT */ - -#ifndef MY_CXT_CLONE -#define MY_CXT_CLONE NOOP -#endif - -#endif - -#ifndef IVdf -# if IVSIZE == LONGSIZE -# define IVdf "ld" -# define UVuf "lu" -# define UVof "lo" -# define UVxf "lx" -# define UVXf "lX" -# else -# if IVSIZE == INTSIZE -# define IVdf "d" -# define UVuf "u" -# define UVof "o" -# define UVxf "x" -# define UVXf "X" -# endif -# endif -#endif - -#ifndef NVef -# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) && \ - defined(PERL_PRIfldbl) && (PERL_BCDVERSION != 0x5006000) - /* Not very likely, but let's try anyway. */ -# define NVef PERL_PRIeldbl -# define NVff PERL_PRIfldbl -# define NVgf PERL_PRIgldbl -# else -# define NVef "e" -# define NVff "f" -# define NVgf "g" -# endif -#endif - -#ifndef SvREFCNT_inc -# ifdef PERL_USE_GCC_BRACE_GROUPS -# define SvREFCNT_inc(sv) \ - ({ \ - SV * const _sv = (SV*)(sv); \ - if (_sv) \ - (SvREFCNT(_sv))++; \ - _sv; \ - }) -# else -# define SvREFCNT_inc(sv) \ - ((PL_Sv=(SV*)(sv)) ? (++(SvREFCNT(PL_Sv)),PL_Sv) : NULL) -# endif -#endif - -#ifndef SvREFCNT_inc_simple -# ifdef PERL_USE_GCC_BRACE_GROUPS -# define SvREFCNT_inc_simple(sv) \ - ({ \ - if (sv) \ - (SvREFCNT(sv))++; \ - (SV *)(sv); \ - }) -# else -# define SvREFCNT_inc_simple(sv) \ - ((sv) ? (SvREFCNT(sv)++,(SV*)(sv)) : NULL) -# endif -#endif - -#ifndef SvREFCNT_inc_NN -# ifdef PERL_USE_GCC_BRACE_GROUPS -# define SvREFCNT_inc_NN(sv) \ - ({ \ - SV * const _sv = (SV*)(sv); \ - SvREFCNT(_sv)++; \ - _sv; \ - }) -# else -# define SvREFCNT_inc_NN(sv) \ - (PL_Sv=(SV*)(sv),++(SvREFCNT(PL_Sv)),PL_Sv) -# endif -#endif - -#ifndef SvREFCNT_inc_void -# ifdef PERL_USE_GCC_BRACE_GROUPS -# define SvREFCNT_inc_void(sv) \ - ({ \ - SV * const _sv = (SV*)(sv); \ - if (_sv) \ - (void)(SvREFCNT(_sv)++); \ - }) -# else -# define SvREFCNT_inc_void(sv) \ - (void)((PL_Sv=(SV*)(sv)) ? ++(SvREFCNT(PL_Sv)) : 0) -# endif -#endif -#ifndef SvREFCNT_inc_simple_void -# define SvREFCNT_inc_simple_void(sv) STMT_START { if (sv) SvREFCNT(sv)++; } STMT_END -#endif - -#ifndef SvREFCNT_inc_simple_NN -# define SvREFCNT_inc_simple_NN(sv) (++SvREFCNT(sv), (SV*)(sv)) -#endif - -#ifndef SvREFCNT_inc_void_NN -# define SvREFCNT_inc_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) -#endif - -#ifndef SvREFCNT_inc_simple_void_NN -# define SvREFCNT_inc_simple_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) -#endif - -#ifndef newSV_type - -#if defined(NEED_newSV_type) -static SV* DPPP_(my_newSV_type)(pTHX_ svtype const t); -static -#else -extern SV* DPPP_(my_newSV_type)(pTHX_ svtype const t); -#endif - -#ifdef newSV_type -# undef newSV_type -#endif -#define newSV_type(a) DPPP_(my_newSV_type)(aTHX_ a) -#define Perl_newSV_type DPPP_(my_newSV_type) - -#if defined(NEED_newSV_type) || defined(NEED_newSV_type_GLOBAL) - -SV* -DPPP_(my_newSV_type)(pTHX_ svtype const t) -{ - SV* const sv = newSV(0); - sv_upgrade(sv, t); - return sv; -} - -#endif - -#endif - -#if (PERL_BCDVERSION < 0x5006000) -# define D_PPP_CONSTPV_ARG(x) ((char *) (x)) -#else -# define D_PPP_CONSTPV_ARG(x) (x) -#endif -#ifndef newSVpvn -# define newSVpvn(data,len) ((data) \ - ? ((len) ? newSVpv((data), (len)) : newSVpv("", 0)) \ - : newSV(0)) -#endif -#ifndef newSVpvn_utf8 -# define newSVpvn_utf8(s, len, u) newSVpvn_flags((s), (len), (u) ? SVf_UTF8 : 0) -#endif -#ifndef SVf_UTF8 -# define SVf_UTF8 0 -#endif - -#ifndef newSVpvn_flags - -#if defined(NEED_newSVpvn_flags) -static SV * DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags); -static -#else -extern SV * DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags); -#endif - -#ifdef newSVpvn_flags -# undef newSVpvn_flags -#endif -#define newSVpvn_flags(a,b,c) DPPP_(my_newSVpvn_flags)(aTHX_ a,b,c) -#define Perl_newSVpvn_flags DPPP_(my_newSVpvn_flags) - -#if defined(NEED_newSVpvn_flags) || defined(NEED_newSVpvn_flags_GLOBAL) - -SV * -DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags) -{ - SV *sv = newSVpvn(D_PPP_CONSTPV_ARG(s), len); - SvFLAGS(sv) |= (flags & SVf_UTF8); - return (flags & SVs_TEMP) ? sv_2mortal(sv) : sv; -} - -#endif - -#endif - -/* Backwards compatibility stuff... :-( */ -#if !defined(NEED_sv_2pv_flags) && defined(NEED_sv_2pv_nolen) -# define NEED_sv_2pv_flags -#endif -#if !defined(NEED_sv_2pv_flags_GLOBAL) && defined(NEED_sv_2pv_nolen_GLOBAL) -# define NEED_sv_2pv_flags_GLOBAL -#endif - -/* Hint: sv_2pv_nolen - * Use the SvPV_nolen() or SvPV_nolen_const() macros instead of sv_2pv_nolen(). - */ -#ifndef sv_2pv_nolen -# define sv_2pv_nolen(sv) SvPV_nolen(sv) -#endif - -#ifdef SvPVbyte - -/* Hint: SvPVbyte - * Does not work in perl-5.6.1, ppport.h implements a version - * borrowed from perl-5.7.3. - */ - -#if (PERL_BCDVERSION < 0x5007000) - -#if defined(NEED_sv_2pvbyte) -static char * DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp); -static -#else -extern char * DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp); -#endif - -#ifdef sv_2pvbyte -# undef sv_2pvbyte -#endif -#define sv_2pvbyte(a,b) DPPP_(my_sv_2pvbyte)(aTHX_ a,b) -#define Perl_sv_2pvbyte DPPP_(my_sv_2pvbyte) - -#if defined(NEED_sv_2pvbyte) || defined(NEED_sv_2pvbyte_GLOBAL) - -char * -DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp) -{ - sv_utf8_downgrade(sv,0); - return SvPV(sv,*lp); -} - -#endif - -/* Hint: sv_2pvbyte - * Use the SvPVbyte() macro instead of sv_2pvbyte(). - */ - -#undef SvPVbyte - -#define SvPVbyte(sv, lp) \ - ((SvFLAGS(sv) & (SVf_POK|SVf_UTF8)) == (SVf_POK) \ - ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_2pvbyte(sv, &lp)) - -#endif - -#else - -# define SvPVbyte SvPV -# define sv_2pvbyte sv_2pv - -#endif -#ifndef sv_2pvbyte_nolen -# define sv_2pvbyte_nolen(sv) sv_2pv_nolen(sv) -#endif - -/* Hint: sv_pvn - * Always use the SvPV() macro instead of sv_pvn(). - */ - -/* Hint: sv_pvn_force - * Always use the SvPV_force() macro instead of sv_pvn_force(). - */ - -/* If these are undefined, they're not handled by the core anyway */ -#ifndef SV_IMMEDIATE_UNREF -# define SV_IMMEDIATE_UNREF 0 -#endif - -#ifndef SV_GMAGIC -# define SV_GMAGIC 0 -#endif - -#ifndef SV_COW_DROP_PV -# define SV_COW_DROP_PV 0 -#endif - -#ifndef SV_UTF8_NO_ENCODING -# define SV_UTF8_NO_ENCODING 0 -#endif - -#ifndef SV_NOSTEAL -# define SV_NOSTEAL 0 -#endif - -#ifndef SV_CONST_RETURN -# define SV_CONST_RETURN 0 -#endif - -#ifndef SV_MUTABLE_RETURN -# define SV_MUTABLE_RETURN 0 -#endif - -#ifndef SV_SMAGIC -# define SV_SMAGIC 0 -#endif - -#ifndef SV_HAS_TRAILING_NUL -# define SV_HAS_TRAILING_NUL 0 -#endif - -#ifndef SV_COW_SHARED_HASH_KEYS -# define SV_COW_SHARED_HASH_KEYS 0 -#endif - -#if (PERL_BCDVERSION < 0x5007002) - -#if defined(NEED_sv_2pv_flags) -static char * DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); -static -#else -extern char * DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); -#endif - -#ifdef sv_2pv_flags -# undef sv_2pv_flags -#endif -#define sv_2pv_flags(a,b,c) DPPP_(my_sv_2pv_flags)(aTHX_ a,b,c) -#define Perl_sv_2pv_flags DPPP_(my_sv_2pv_flags) - -#if defined(NEED_sv_2pv_flags) || defined(NEED_sv_2pv_flags_GLOBAL) - -char * -DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags) -{ - STRLEN n_a = (STRLEN) flags; - return sv_2pv(sv, lp ? lp : &n_a); -} - -#endif - -#if defined(NEED_sv_pvn_force_flags) -static char * DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); -static -#else -extern char * DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); -#endif - -#ifdef sv_pvn_force_flags -# undef sv_pvn_force_flags -#endif -#define sv_pvn_force_flags(a,b,c) DPPP_(my_sv_pvn_force_flags)(aTHX_ a,b,c) -#define Perl_sv_pvn_force_flags DPPP_(my_sv_pvn_force_flags) - -#if defined(NEED_sv_pvn_force_flags) || defined(NEED_sv_pvn_force_flags_GLOBAL) - -char * -DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags) -{ - STRLEN n_a = (STRLEN) flags; - return sv_pvn_force(sv, lp ? lp : &n_a); -} - -#endif - -#endif - -#if (PERL_BCDVERSION < 0x5008008) || ( (PERL_BCDVERSION >= 0x5009000) && (PERL_BCDVERSION < 0x5009003) ) -# define DPPP_SVPV_NOLEN_LP_ARG &PL_na -#else -# define DPPP_SVPV_NOLEN_LP_ARG 0 -#endif -#ifndef SvPV_const -# define SvPV_const(sv, lp) SvPV_flags_const(sv, lp, SV_GMAGIC) -#endif - -#ifndef SvPV_mutable -# define SvPV_mutable(sv, lp) SvPV_flags_mutable(sv, lp, SV_GMAGIC) -#endif -#ifndef SvPV_flags -# define SvPV_flags(sv, lp, flags) \ - ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ - ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_2pv_flags(sv, &lp, flags)) -#endif -#ifndef SvPV_flags_const -# define SvPV_flags_const(sv, lp, flags) \ - ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ - ? ((lp = SvCUR(sv)), SvPVX_const(sv)) : \ - (const char*) sv_2pv_flags(sv, &lp, flags|SV_CONST_RETURN)) -#endif -#ifndef SvPV_flags_const_nolen -# define SvPV_flags_const_nolen(sv, flags) \ - ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ - ? SvPVX_const(sv) : \ - (const char*) sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, flags|SV_CONST_RETURN)) -#endif -#ifndef SvPV_flags_mutable -# define SvPV_flags_mutable(sv, lp, flags) \ - ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ - ? ((lp = SvCUR(sv)), SvPVX_mutable(sv)) : \ - sv_2pv_flags(sv, &lp, flags|SV_MUTABLE_RETURN)) -#endif -#ifndef SvPV_force -# define SvPV_force(sv, lp) SvPV_force_flags(sv, lp, SV_GMAGIC) -#endif - -#ifndef SvPV_force_nolen -# define SvPV_force_nolen(sv) SvPV_force_flags_nolen(sv, SV_GMAGIC) -#endif - -#ifndef SvPV_force_mutable -# define SvPV_force_mutable(sv, lp) SvPV_force_flags_mutable(sv, lp, SV_GMAGIC) -#endif - -#ifndef SvPV_force_nomg -# define SvPV_force_nomg(sv, lp) SvPV_force_flags(sv, lp, 0) -#endif - -#ifndef SvPV_force_nomg_nolen -# define SvPV_force_nomg_nolen(sv) SvPV_force_flags_nolen(sv, 0) -#endif -#ifndef SvPV_force_flags -# define SvPV_force_flags(sv, lp, flags) \ - ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ - ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_pvn_force_flags(sv, &lp, flags)) -#endif -#ifndef SvPV_force_flags_nolen -# define SvPV_force_flags_nolen(sv, flags) \ - ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ - ? SvPVX(sv) : sv_pvn_force_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, flags)) -#endif -#ifndef SvPV_force_flags_mutable -# define SvPV_force_flags_mutable(sv, lp, flags) \ - ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ - ? ((lp = SvCUR(sv)), SvPVX_mutable(sv)) \ - : sv_pvn_force_flags(sv, &lp, flags|SV_MUTABLE_RETURN)) -#endif -#ifndef SvPV_nolen -# define SvPV_nolen(sv) \ - ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ - ? SvPVX(sv) : sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, SV_GMAGIC)) -#endif -#ifndef SvPV_nolen_const -# define SvPV_nolen_const(sv) \ - ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ - ? SvPVX_const(sv) : sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, SV_GMAGIC|SV_CONST_RETURN)) -#endif -#ifndef SvPV_nomg -# define SvPV_nomg(sv, lp) SvPV_flags(sv, lp, 0) -#endif - -#ifndef SvPV_nomg_const -# define SvPV_nomg_const(sv, lp) SvPV_flags_const(sv, lp, 0) -#endif - -#ifndef SvPV_nomg_const_nolen -# define SvPV_nomg_const_nolen(sv) SvPV_flags_const_nolen(sv, 0) -#endif -#ifndef SvPV_renew -# define SvPV_renew(sv,n) STMT_START { SvLEN_set(sv, n); \ - SvPV_set((sv), (char *) saferealloc( \ - (Malloc_t)SvPVX(sv), (MEM_SIZE)((n)))); \ - } STMT_END -#endif -#ifndef SvMAGIC_set -# define SvMAGIC_set(sv, val) \ - STMT_START { assert(SvTYPE(sv) >= SVt_PVMG); \ - (((XPVMG*) SvANY(sv))->xmg_magic = (val)); } STMT_END -#endif - -#if (PERL_BCDVERSION < 0x5009003) -#ifndef SvPVX_const -# define SvPVX_const(sv) ((const char*) (0 + SvPVX(sv))) -#endif - -#ifndef SvPVX_mutable -# define SvPVX_mutable(sv) (0 + SvPVX(sv)) -#endif -#ifndef SvRV_set -# define SvRV_set(sv, val) \ - STMT_START { assert(SvTYPE(sv) >= SVt_RV); \ - (((XRV*) SvANY(sv))->xrv_rv = (val)); } STMT_END -#endif - -#else -#ifndef SvPVX_const -# define SvPVX_const(sv) ((const char*)((sv)->sv_u.svu_pv)) -#endif - -#ifndef SvPVX_mutable -# define SvPVX_mutable(sv) ((sv)->sv_u.svu_pv) -#endif -#ifndef SvRV_set -# define SvRV_set(sv, val) \ - STMT_START { assert(SvTYPE(sv) >= SVt_RV); \ - ((sv)->sv_u.svu_rv = (val)); } STMT_END -#endif - -#endif -#ifndef SvSTASH_set -# define SvSTASH_set(sv, val) \ - STMT_START { assert(SvTYPE(sv) >= SVt_PVMG); \ - (((XPVMG*) SvANY(sv))->xmg_stash = (val)); } STMT_END -#endif - -#if (PERL_BCDVERSION < 0x5004000) -#ifndef SvUV_set -# define SvUV_set(sv, val) \ - STMT_START { assert(SvTYPE(sv) == SVt_IV || SvTYPE(sv) >= SVt_PVIV); \ - (((XPVIV*) SvANY(sv))->xiv_iv = (IV) (val)); } STMT_END -#endif - -#else -#ifndef SvUV_set -# define SvUV_set(sv, val) \ - STMT_START { assert(SvTYPE(sv) == SVt_IV || SvTYPE(sv) >= SVt_PVIV); \ - (((XPVUV*) SvANY(sv))->xuv_uv = (val)); } STMT_END -#endif - -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(vnewSVpvf) -#if defined(NEED_vnewSVpvf) -static SV * DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args); -static -#else -extern SV * DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args); -#endif - -#ifdef vnewSVpvf -# undef vnewSVpvf -#endif -#define vnewSVpvf(a,b) DPPP_(my_vnewSVpvf)(aTHX_ a,b) -#define Perl_vnewSVpvf DPPP_(my_vnewSVpvf) - -#if defined(NEED_vnewSVpvf) || defined(NEED_vnewSVpvf_GLOBAL) - -SV * -DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args) -{ - register SV *sv = newSV(0); - sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); - return sv; -} - -#endif -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vcatpvf) -# define sv_vcatpvf(sv, pat, args) sv_vcatpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)) -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vsetpvf) -# define sv_vsetpvf(sv, pat, args) sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)) -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_catpvf_mg) -#if defined(NEED_sv_catpvf_mg) -static void DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...); -static -#else -extern void DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...); -#endif - -#define Perl_sv_catpvf_mg DPPP_(my_sv_catpvf_mg) - -#if defined(NEED_sv_catpvf_mg) || defined(NEED_sv_catpvf_mg_GLOBAL) - -void -DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...) -{ - va_list args; - va_start(args, pat); - sv_vcatpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); - SvSETMAGIC(sv); - va_end(args); -} - -#endif -#endif - -#ifdef PERL_IMPLICIT_CONTEXT -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_catpvf_mg_nocontext) -#if defined(NEED_sv_catpvf_mg_nocontext) -static void DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...); -static -#else -extern void DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...); -#endif - -#define sv_catpvf_mg_nocontext DPPP_(my_sv_catpvf_mg_nocontext) -#define Perl_sv_catpvf_mg_nocontext DPPP_(my_sv_catpvf_mg_nocontext) - -#if defined(NEED_sv_catpvf_mg_nocontext) || defined(NEED_sv_catpvf_mg_nocontext_GLOBAL) - -void -DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...) -{ - dTHX; - va_list args; - va_start(args, pat); - sv_vcatpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); - SvSETMAGIC(sv); - va_end(args); -} - -#endif -#endif -#endif - -/* sv_catpvf_mg depends on sv_catpvf_mg_nocontext */ -#ifndef sv_catpvf_mg -# ifdef PERL_IMPLICIT_CONTEXT -# define sv_catpvf_mg Perl_sv_catpvf_mg_nocontext -# else -# define sv_catpvf_mg Perl_sv_catpvf_mg -# endif -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vcatpvf_mg) -# define sv_vcatpvf_mg(sv, pat, args) \ - STMT_START { \ - sv_vcatpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); \ - SvSETMAGIC(sv); \ - } STMT_END -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_setpvf_mg) -#if defined(NEED_sv_setpvf_mg) -static void DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...); -static -#else -extern void DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...); -#endif - -#define Perl_sv_setpvf_mg DPPP_(my_sv_setpvf_mg) - -#if defined(NEED_sv_setpvf_mg) || defined(NEED_sv_setpvf_mg_GLOBAL) - -void -DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...) -{ - va_list args; - va_start(args, pat); - sv_vsetpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); - SvSETMAGIC(sv); - va_end(args); -} - -#endif -#endif - -#ifdef PERL_IMPLICIT_CONTEXT -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_setpvf_mg_nocontext) -#if defined(NEED_sv_setpvf_mg_nocontext) -static void DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...); -static -#else -extern void DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...); -#endif - -#define sv_setpvf_mg_nocontext DPPP_(my_sv_setpvf_mg_nocontext) -#define Perl_sv_setpvf_mg_nocontext DPPP_(my_sv_setpvf_mg_nocontext) - -#if defined(NEED_sv_setpvf_mg_nocontext) || defined(NEED_sv_setpvf_mg_nocontext_GLOBAL) - -void -DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...) -{ - dTHX; - va_list args; - va_start(args, pat); - sv_vsetpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); - SvSETMAGIC(sv); - va_end(args); -} - -#endif -#endif -#endif - -/* sv_setpvf_mg depends on sv_setpvf_mg_nocontext */ -#ifndef sv_setpvf_mg -# ifdef PERL_IMPLICIT_CONTEXT -# define sv_setpvf_mg Perl_sv_setpvf_mg_nocontext -# else -# define sv_setpvf_mg Perl_sv_setpvf_mg -# endif -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vsetpvf_mg) -# define sv_vsetpvf_mg(sv, pat, args) \ - STMT_START { \ - sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); \ - SvSETMAGIC(sv); \ - } STMT_END -#endif - -#ifndef newSVpvn_share - -#if defined(NEED_newSVpvn_share) -static SV * DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash); -static -#else -extern SV * DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash); -#endif - -#ifdef newSVpvn_share -# undef newSVpvn_share -#endif -#define newSVpvn_share(a,b,c) DPPP_(my_newSVpvn_share)(aTHX_ a,b,c) -#define Perl_newSVpvn_share DPPP_(my_newSVpvn_share) - -#if defined(NEED_newSVpvn_share) || defined(NEED_newSVpvn_share_GLOBAL) - -SV * -DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash) -{ - SV *sv; - if (len < 0) - len = -len; - if (!hash) - PERL_HASH(hash, (char*) src, len); - sv = newSVpvn((char *) src, len); - sv_upgrade(sv, SVt_PVIV); - SvIVX(sv) = hash; - SvREADONLY_on(sv); - SvPOK_on(sv); - return sv; -} - -#endif - -#endif -#ifndef SvSHARED_HASH -# define SvSHARED_HASH(sv) (0 + SvUVX(sv)) -#endif -#ifndef HvNAME_get -# define HvNAME_get(hv) HvNAME(hv) -#endif -#ifndef HvNAMELEN_get -# define HvNAMELEN_get(hv) (HvNAME_get(hv) ? (I32)strlen(HvNAME_get(hv)) : 0) -#endif -#ifndef GvSVn -# define GvSVn(gv) GvSV(gv) -#endif - -#ifndef isGV_with_GP -# define isGV_with_GP(gv) isGV(gv) -#endif -#ifndef WARN_ALL -# define WARN_ALL 0 -#endif - -#ifndef WARN_CLOSURE -# define WARN_CLOSURE 1 -#endif - -#ifndef WARN_DEPRECATED -# define WARN_DEPRECATED 2 -#endif - -#ifndef WARN_EXITING -# define WARN_EXITING 3 -#endif - -#ifndef WARN_GLOB -# define WARN_GLOB 4 -#endif - -#ifndef WARN_IO -# define WARN_IO 5 -#endif - -#ifndef WARN_CLOSED -# define WARN_CLOSED 6 -#endif - -#ifndef WARN_EXEC -# define WARN_EXEC 7 -#endif - -#ifndef WARN_LAYER -# define WARN_LAYER 8 -#endif - -#ifndef WARN_NEWLINE -# define WARN_NEWLINE 9 -#endif - -#ifndef WARN_PIPE -# define WARN_PIPE 10 -#endif - -#ifndef WARN_UNOPENED -# define WARN_UNOPENED 11 -#endif - -#ifndef WARN_MISC -# define WARN_MISC 12 -#endif - -#ifndef WARN_NUMERIC -# define WARN_NUMERIC 13 -#endif - -#ifndef WARN_ONCE -# define WARN_ONCE 14 -#endif - -#ifndef WARN_OVERFLOW -# define WARN_OVERFLOW 15 -#endif - -#ifndef WARN_PACK -# define WARN_PACK 16 -#endif - -#ifndef WARN_PORTABLE -# define WARN_PORTABLE 17 -#endif - -#ifndef WARN_RECURSION -# define WARN_RECURSION 18 -#endif - -#ifndef WARN_REDEFINE -# define WARN_REDEFINE 19 -#endif - -#ifndef WARN_REGEXP -# define WARN_REGEXP 20 -#endif - -#ifndef WARN_SEVERE -# define WARN_SEVERE 21 -#endif - -#ifndef WARN_DEBUGGING -# define WARN_DEBUGGING 22 -#endif - -#ifndef WARN_INPLACE -# define WARN_INPLACE 23 -#endif - -#ifndef WARN_INTERNAL -# define WARN_INTERNAL 24 -#endif - -#ifndef WARN_MALLOC -# define WARN_MALLOC 25 -#endif - -#ifndef WARN_SIGNAL -# define WARN_SIGNAL 26 -#endif - -#ifndef WARN_SUBSTR -# define WARN_SUBSTR 27 -#endif - -#ifndef WARN_SYNTAX -# define WARN_SYNTAX 28 -#endif - -#ifndef WARN_AMBIGUOUS -# define WARN_AMBIGUOUS 29 -#endif - -#ifndef WARN_BAREWORD -# define WARN_BAREWORD 30 -#endif - -#ifndef WARN_DIGIT -# define WARN_DIGIT 31 -#endif - -#ifndef WARN_PARENTHESIS -# define WARN_PARENTHESIS 32 -#endif - -#ifndef WARN_PRECEDENCE -# define WARN_PRECEDENCE 33 -#endif - -#ifndef WARN_PRINTF -# define WARN_PRINTF 34 -#endif - -#ifndef WARN_PROTOTYPE -# define WARN_PROTOTYPE 35 -#endif - -#ifndef WARN_QW -# define WARN_QW 36 -#endif - -#ifndef WARN_RESERVED -# define WARN_RESERVED 37 -#endif - -#ifndef WARN_SEMICOLON -# define WARN_SEMICOLON 38 -#endif - -#ifndef WARN_TAINT -# define WARN_TAINT 39 -#endif - -#ifndef WARN_THREADS -# define WARN_THREADS 40 -#endif - -#ifndef WARN_UNINITIALIZED -# define WARN_UNINITIALIZED 41 -#endif - -#ifndef WARN_UNPACK -# define WARN_UNPACK 42 -#endif - -#ifndef WARN_UNTIE -# define WARN_UNTIE 43 -#endif - -#ifndef WARN_UTF8 -# define WARN_UTF8 44 -#endif - -#ifndef WARN_VOID -# define WARN_VOID 45 -#endif - -#ifndef WARN_ASSERTIONS -# define WARN_ASSERTIONS 46 -#endif -#ifndef packWARN -# define packWARN(a) (a) -#endif - -#ifndef ckWARN -# ifdef G_WARN_ON -# define ckWARN(a) (PL_dowarn & G_WARN_ON) -# else -# define ckWARN(a) PL_dowarn -# endif -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(warner) -#if defined(NEED_warner) -static void DPPP_(my_warner)(U32 err, const char *pat, ...); -static -#else -extern void DPPP_(my_warner)(U32 err, const char *pat, ...); -#endif - -#define Perl_warner DPPP_(my_warner) - -#if defined(NEED_warner) || defined(NEED_warner_GLOBAL) - -void -DPPP_(my_warner)(U32 err, const char *pat, ...) -{ - SV *sv; - va_list args; - - PERL_UNUSED_ARG(err); - - va_start(args, pat); - sv = vnewSVpvf(pat, &args); - va_end(args); - sv_2mortal(sv); - warn("%s", SvPV_nolen(sv)); -} - -#define warner Perl_warner - -#define Perl_warner_nocontext Perl_warner - -#endif -#endif - -/* concatenating with "" ensures that only literal strings are accepted as argument - * note that STR_WITH_LEN() can't be used as argument to macros or functions that - * under some configurations might be macros - */ -#ifndef STR_WITH_LEN -# define STR_WITH_LEN(s) (s ""), (sizeof(s)-1) -#endif -#ifndef newSVpvs -# define newSVpvs(str) newSVpvn(str "", sizeof(str) - 1) -#endif - -#ifndef newSVpvs_flags -# define newSVpvs_flags(str, flags) newSVpvn_flags(str "", sizeof(str) - 1, flags) -#endif - -#ifndef sv_catpvs -# define sv_catpvs(sv, str) sv_catpvn(sv, str "", sizeof(str) - 1) -#endif - -#ifndef sv_setpvs -# define sv_setpvs(sv, str) sv_setpvn(sv, str "", sizeof(str) - 1) -#endif - -#ifndef hv_fetchs -# define hv_fetchs(hv, key, lval) hv_fetch(hv, key "", sizeof(key) - 1, lval) -#endif - -#ifndef hv_stores -# define hv_stores(hv, key, val) hv_store(hv, key "", sizeof(key) - 1, val, 0) -#endif -#ifndef gv_fetchpvn_flags -# define gv_fetchpvn_flags(name, len, flags, svt) gv_fetchpv(name, flags, svt) -#endif - -#ifndef gv_fetchpvs -# define gv_fetchpvs(name, flags, svt) gv_fetchpvn_flags(name "", sizeof(name) - 1, flags, svt) -#endif - -#ifndef gv_stashpvs -# define gv_stashpvs(name, flags) gv_stashpvn(name "", sizeof(name) - 1, flags) -#endif -#ifndef SvGETMAGIC -# define SvGETMAGIC(x) STMT_START { if (SvGMAGICAL(x)) mg_get(x); } STMT_END -#endif -#ifndef PERL_MAGIC_sv -# define PERL_MAGIC_sv '\0' -#endif - -#ifndef PERL_MAGIC_overload -# define PERL_MAGIC_overload 'A' -#endif - -#ifndef PERL_MAGIC_overload_elem -# define PERL_MAGIC_overload_elem 'a' -#endif - -#ifndef PERL_MAGIC_overload_table -# define PERL_MAGIC_overload_table 'c' -#endif - -#ifndef PERL_MAGIC_bm -# define PERL_MAGIC_bm 'B' -#endif - -#ifndef PERL_MAGIC_regdata -# define PERL_MAGIC_regdata 'D' -#endif - -#ifndef PERL_MAGIC_regdatum -# define PERL_MAGIC_regdatum 'd' -#endif - -#ifndef PERL_MAGIC_env -# define PERL_MAGIC_env 'E' -#endif - -#ifndef PERL_MAGIC_envelem -# define PERL_MAGIC_envelem 'e' -#endif - -#ifndef PERL_MAGIC_fm -# define PERL_MAGIC_fm 'f' -#endif - -#ifndef PERL_MAGIC_regex_global -# define PERL_MAGIC_regex_global 'g' -#endif - -#ifndef PERL_MAGIC_isa -# define PERL_MAGIC_isa 'I' -#endif - -#ifndef PERL_MAGIC_isaelem -# define PERL_MAGIC_isaelem 'i' -#endif - -#ifndef PERL_MAGIC_nkeys -# define PERL_MAGIC_nkeys 'k' -#endif - -#ifndef PERL_MAGIC_dbfile -# define PERL_MAGIC_dbfile 'L' -#endif - -#ifndef PERL_MAGIC_dbline -# define PERL_MAGIC_dbline 'l' -#endif - -#ifndef PERL_MAGIC_mutex -# define PERL_MAGIC_mutex 'm' -#endif - -#ifndef PERL_MAGIC_shared -# define PERL_MAGIC_shared 'N' -#endif - -#ifndef PERL_MAGIC_shared_scalar -# define PERL_MAGIC_shared_scalar 'n' -#endif - -#ifndef PERL_MAGIC_collxfrm -# define PERL_MAGIC_collxfrm 'o' -#endif - -#ifndef PERL_MAGIC_tied -# define PERL_MAGIC_tied 'P' -#endif - -#ifndef PERL_MAGIC_tiedelem -# define PERL_MAGIC_tiedelem 'p' -#endif - -#ifndef PERL_MAGIC_tiedscalar -# define PERL_MAGIC_tiedscalar 'q' -#endif - -#ifndef PERL_MAGIC_qr -# define PERL_MAGIC_qr 'r' -#endif - -#ifndef PERL_MAGIC_sig -# define PERL_MAGIC_sig 'S' -#endif - -#ifndef PERL_MAGIC_sigelem -# define PERL_MAGIC_sigelem 's' -#endif - -#ifndef PERL_MAGIC_taint -# define PERL_MAGIC_taint 't' -#endif - -#ifndef PERL_MAGIC_uvar -# define PERL_MAGIC_uvar 'U' -#endif - -#ifndef PERL_MAGIC_uvar_elem -# define PERL_MAGIC_uvar_elem 'u' -#endif - -#ifndef PERL_MAGIC_vstring -# define PERL_MAGIC_vstring 'V' -#endif - -#ifndef PERL_MAGIC_vec -# define PERL_MAGIC_vec 'v' -#endif - -#ifndef PERL_MAGIC_utf8 -# define PERL_MAGIC_utf8 'w' -#endif - -#ifndef PERL_MAGIC_substr -# define PERL_MAGIC_substr 'x' -#endif - -#ifndef PERL_MAGIC_defelem -# define PERL_MAGIC_defelem 'y' -#endif - -#ifndef PERL_MAGIC_glob -# define PERL_MAGIC_glob '*' -#endif - -#ifndef PERL_MAGIC_arylen -# define PERL_MAGIC_arylen '#' -#endif - -#ifndef PERL_MAGIC_pos -# define PERL_MAGIC_pos '.' -#endif - -#ifndef PERL_MAGIC_backref -# define PERL_MAGIC_backref '<' -#endif - -#ifndef PERL_MAGIC_ext -# define PERL_MAGIC_ext '~' -#endif - -/* That's the best we can do... */ -#ifndef sv_catpvn_nomg -# define sv_catpvn_nomg sv_catpvn -#endif - -#ifndef sv_catsv_nomg -# define sv_catsv_nomg sv_catsv -#endif - -#ifndef sv_setsv_nomg -# define sv_setsv_nomg sv_setsv -#endif - -#ifndef sv_pvn_nomg -# define sv_pvn_nomg sv_pvn -#endif - -#ifndef SvIV_nomg -# define SvIV_nomg SvIV -#endif - -#ifndef SvUV_nomg -# define SvUV_nomg SvUV -#endif - -#ifndef sv_catpv_mg -# define sv_catpv_mg(sv, ptr) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_catpv(TeMpSv,ptr); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_catpvn_mg -# define sv_catpvn_mg(sv, ptr, len) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_catpvn(TeMpSv,ptr,len); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_catsv_mg -# define sv_catsv_mg(dsv, ssv) \ - STMT_START { \ - SV *TeMpSv = dsv; \ - sv_catsv(TeMpSv,ssv); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_setiv_mg -# define sv_setiv_mg(sv, i) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_setiv(TeMpSv,i); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_setnv_mg -# define sv_setnv_mg(sv, num) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_setnv(TeMpSv,num); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_setpv_mg -# define sv_setpv_mg(sv, ptr) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_setpv(TeMpSv,ptr); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_setpvn_mg -# define sv_setpvn_mg(sv, ptr, len) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_setpvn(TeMpSv,ptr,len); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_setsv_mg -# define sv_setsv_mg(dsv, ssv) \ - STMT_START { \ - SV *TeMpSv = dsv; \ - sv_setsv(TeMpSv,ssv); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_setuv_mg -# define sv_setuv_mg(sv, i) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_setuv(TeMpSv,i); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_usepvn_mg -# define sv_usepvn_mg(sv, ptr, len) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_usepvn(TeMpSv,ptr,len); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif -#ifndef SvVSTRING_mg -# define SvVSTRING_mg(sv) (SvMAGICAL(sv) ? mg_find(sv, PERL_MAGIC_vstring) : NULL) -#endif - -/* Hint: sv_magic_portable - * This is a compatibility function that is only available with - * Devel::PPPort. It is NOT in the perl core. - * Its purpose is to mimic the 5.8.0 behaviour of sv_magic() when - * it is being passed a name pointer with namlen == 0. In that - * case, perl 5.8.0 and later store the pointer, not a copy of it. - * The compatibility can be provided back to perl 5.004. With - * earlier versions, the code will not compile. - */ - -#if (PERL_BCDVERSION < 0x5004000) - - /* code that uses sv_magic_portable will not compile */ - -#elif (PERL_BCDVERSION < 0x5008000) - -# define sv_magic_portable(sv, obj, how, name, namlen) \ - STMT_START { \ - SV *SvMp_sv = (sv); \ - char *SvMp_name = (char *) (name); \ - I32 SvMp_namlen = (namlen); \ - if (SvMp_name && SvMp_namlen == 0) \ - { \ - MAGIC *mg; \ - sv_magic(SvMp_sv, obj, how, 0, 0); \ - mg = SvMAGIC(SvMp_sv); \ - mg->mg_len = -42; /* XXX: this is the tricky part */ \ - mg->mg_ptr = SvMp_name; \ - } \ - else \ - { \ - sv_magic(SvMp_sv, obj, how, SvMp_name, SvMp_namlen); \ - } \ - } STMT_END - -#else - -# define sv_magic_portable(a, b, c, d, e) sv_magic(a, b, c, d, e) - -#endif - -#ifdef USE_ITHREADS -#ifndef CopFILE -# define CopFILE(c) ((c)->cop_file) -#endif - -#ifndef CopFILEGV -# define CopFILEGV(c) (CopFILE(c) ? gv_fetchfile(CopFILE(c)) : Nullgv) -#endif - -#ifndef CopFILE_set -# define CopFILE_set(c,pv) ((c)->cop_file = savepv(pv)) -#endif - -#ifndef CopFILESV -# define CopFILESV(c) (CopFILE(c) ? GvSV(gv_fetchfile(CopFILE(c))) : Nullsv) -#endif - -#ifndef CopFILEAV -# define CopFILEAV(c) (CopFILE(c) ? GvAV(gv_fetchfile(CopFILE(c))) : Nullav) -#endif - -#ifndef CopSTASHPV -# define CopSTASHPV(c) ((c)->cop_stashpv) -#endif - -#ifndef CopSTASHPV_set -# define CopSTASHPV_set(c,pv) ((c)->cop_stashpv = ((pv) ? savepv(pv) : Nullch)) -#endif - -#ifndef CopSTASH -# define CopSTASH(c) (CopSTASHPV(c) ? gv_stashpv(CopSTASHPV(c),GV_ADD) : Nullhv) -#endif - -#ifndef CopSTASH_set -# define CopSTASH_set(c,hv) CopSTASHPV_set(c, (hv) ? HvNAME(hv) : Nullch) -#endif - -#ifndef CopSTASH_eq -# define CopSTASH_eq(c,hv) ((hv) && (CopSTASHPV(c) == HvNAME(hv) \ - || (CopSTASHPV(c) && HvNAME(hv) \ - && strEQ(CopSTASHPV(c), HvNAME(hv))))) -#endif - -#else -#ifndef CopFILEGV -# define CopFILEGV(c) ((c)->cop_filegv) -#endif - -#ifndef CopFILEGV_set -# define CopFILEGV_set(c,gv) ((c)->cop_filegv = (GV*)SvREFCNT_inc(gv)) -#endif - -#ifndef CopFILE_set -# define CopFILE_set(c,pv) CopFILEGV_set((c), gv_fetchfile(pv)) -#endif - -#ifndef CopFILESV -# define CopFILESV(c) (CopFILEGV(c) ? GvSV(CopFILEGV(c)) : Nullsv) -#endif - -#ifndef CopFILEAV -# define CopFILEAV(c) (CopFILEGV(c) ? GvAV(CopFILEGV(c)) : Nullav) -#endif - -#ifndef CopFILE -# define CopFILE(c) (CopFILESV(c) ? SvPVX(CopFILESV(c)) : Nullch) -#endif - -#ifndef CopSTASH -# define CopSTASH(c) ((c)->cop_stash) -#endif - -#ifndef CopSTASH_set -# define CopSTASH_set(c,hv) ((c)->cop_stash = (hv)) -#endif - -#ifndef CopSTASHPV -# define CopSTASHPV(c) (CopSTASH(c) ? HvNAME(CopSTASH(c)) : Nullch) -#endif - -#ifndef CopSTASHPV_set -# define CopSTASHPV_set(c,pv) CopSTASH_set((c), gv_stashpv(pv,GV_ADD)) -#endif - -#ifndef CopSTASH_eq -# define CopSTASH_eq(c,hv) (CopSTASH(c) == (hv)) -#endif - -#endif /* USE_ITHREADS */ -#ifndef IN_PERL_COMPILETIME -# define IN_PERL_COMPILETIME (PL_curcop == &PL_compiling) -#endif - -#ifndef IN_LOCALE_RUNTIME -# define IN_LOCALE_RUNTIME (PL_curcop->op_private & HINT_LOCALE) -#endif - -#ifndef IN_LOCALE_COMPILETIME -# define IN_LOCALE_COMPILETIME (PL_hints & HINT_LOCALE) -#endif - -#ifndef IN_LOCALE -# define IN_LOCALE (IN_PERL_COMPILETIME ? IN_LOCALE_COMPILETIME : IN_LOCALE_RUNTIME) -#endif -#ifndef IS_NUMBER_IN_UV -# define IS_NUMBER_IN_UV 0x01 -#endif - -#ifndef IS_NUMBER_GREATER_THAN_UV_MAX -# define IS_NUMBER_GREATER_THAN_UV_MAX 0x02 -#endif - -#ifndef IS_NUMBER_NOT_INT -# define IS_NUMBER_NOT_INT 0x04 -#endif - -#ifndef IS_NUMBER_NEG -# define IS_NUMBER_NEG 0x08 -#endif - -#ifndef IS_NUMBER_INFINITY -# define IS_NUMBER_INFINITY 0x10 -#endif - -#ifndef IS_NUMBER_NAN -# define IS_NUMBER_NAN 0x20 -#endif -#ifndef GROK_NUMERIC_RADIX -# define GROK_NUMERIC_RADIX(sp, send) grok_numeric_radix(sp, send) -#endif -#ifndef PERL_SCAN_GREATER_THAN_UV_MAX -# define PERL_SCAN_GREATER_THAN_UV_MAX 0x02 -#endif - -#ifndef PERL_SCAN_SILENT_ILLDIGIT -# define PERL_SCAN_SILENT_ILLDIGIT 0x04 -#endif - -#ifndef PERL_SCAN_ALLOW_UNDERSCORES -# define PERL_SCAN_ALLOW_UNDERSCORES 0x01 -#endif - -#ifndef PERL_SCAN_DISALLOW_PREFIX -# define PERL_SCAN_DISALLOW_PREFIX 0x02 -#endif - -#ifndef grok_numeric_radix -#if defined(NEED_grok_numeric_radix) -static bool DPPP_(my_grok_numeric_radix)(pTHX_ const char ** sp, const char * send); -static -#else -extern bool DPPP_(my_grok_numeric_radix)(pTHX_ const char ** sp, const char * send); -#endif - -#ifdef grok_numeric_radix -# undef grok_numeric_radix -#endif -#define grok_numeric_radix(a,b) DPPP_(my_grok_numeric_radix)(aTHX_ a,b) -#define Perl_grok_numeric_radix DPPP_(my_grok_numeric_radix) - -#if defined(NEED_grok_numeric_radix) || defined(NEED_grok_numeric_radix_GLOBAL) -bool -DPPP_(my_grok_numeric_radix)(pTHX_ const char **sp, const char *send) -{ -#ifdef USE_LOCALE_NUMERIC -#ifdef PL_numeric_radix_sv - if (PL_numeric_radix_sv && IN_LOCALE) { - STRLEN len; - char* radix = SvPV(PL_numeric_radix_sv, len); - if (*sp + len <= send && memEQ(*sp, radix, len)) { - *sp += len; - return TRUE; - } - } -#else - /* older perls don't have PL_numeric_radix_sv so the radix - * must manually be requested from locale.h - */ -#include - dTHR; /* needed for older threaded perls */ - struct lconv *lc = localeconv(); - char *radix = lc->decimal_point; - if (radix && IN_LOCALE) { - STRLEN len = strlen(radix); - if (*sp + len <= send && memEQ(*sp, radix, len)) { - *sp += len; - return TRUE; - } - } -#endif -#endif /* USE_LOCALE_NUMERIC */ - /* always try "." if numeric radix didn't match because - * we may have data from different locales mixed */ - if (*sp < send && **sp == '.') { - ++*sp; - return TRUE; - } - return FALSE; -} -#endif -#endif - -#ifndef grok_number -#if defined(NEED_grok_number) -static int DPPP_(my_grok_number)(pTHX_ const char * pv, STRLEN len, UV * valuep); -static -#else -extern int DPPP_(my_grok_number)(pTHX_ const char * pv, STRLEN len, UV * valuep); -#endif - -#ifdef grok_number -# undef grok_number -#endif -#define grok_number(a,b,c) DPPP_(my_grok_number)(aTHX_ a,b,c) -#define Perl_grok_number DPPP_(my_grok_number) - -#if defined(NEED_grok_number) || defined(NEED_grok_number_GLOBAL) -int -DPPP_(my_grok_number)(pTHX_ const char *pv, STRLEN len, UV *valuep) -{ - const char *s = pv; - const char *send = pv + len; - const UV max_div_10 = UV_MAX / 10; - const char max_mod_10 = UV_MAX % 10; - int numtype = 0; - int sawinf = 0; - int sawnan = 0; - - while (s < send && isSPACE(*s)) - s++; - if (s == send) { - return 0; - } else if (*s == '-') { - s++; - numtype = IS_NUMBER_NEG; - } - else if (*s == '+') - s++; - - if (s == send) - return 0; - - /* next must be digit or the radix separator or beginning of infinity */ - if (isDIGIT(*s)) { - /* UVs are at least 32 bits, so the first 9 decimal digits cannot - overflow. */ - UV value = *s - '0'; - /* This construction seems to be more optimiser friendly. - (without it gcc does the isDIGIT test and the *s - '0' separately) - With it gcc on arm is managing 6 instructions (6 cycles) per digit. - In theory the optimiser could deduce how far to unroll the loop - before checking for overflow. */ - if (++s < send) { - int digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - /* Now got 9 digits, so need to check - each time for overflow. */ - digit = *s - '0'; - while (digit >= 0 && digit <= 9 - && (value < max_div_10 - || (value == max_div_10 - && digit <= max_mod_10))) { - value = value * 10 + digit; - if (++s < send) - digit = *s - '0'; - else - break; - } - if (digit >= 0 && digit <= 9 - && (s < send)) { - /* value overflowed. - skip the remaining digits, don't - worry about setting *valuep. */ - do { - s++; - } while (s < send && isDIGIT(*s)); - numtype |= - IS_NUMBER_GREATER_THAN_UV_MAX; - goto skip_value; - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - numtype |= IS_NUMBER_IN_UV; - if (valuep) - *valuep = value; - - skip_value: - if (GROK_NUMERIC_RADIX(&s, send)) { - numtype |= IS_NUMBER_NOT_INT; - while (s < send && isDIGIT(*s)) /* optional digits after the radix */ - s++; - } - } - else if (GROK_NUMERIC_RADIX(&s, send)) { - numtype |= IS_NUMBER_NOT_INT | IS_NUMBER_IN_UV; /* valuep assigned below */ - /* no digits before the radix means we need digits after it */ - if (s < send && isDIGIT(*s)) { - do { - s++; - } while (s < send && isDIGIT(*s)); - if (valuep) { - /* integer approximation is valid - it's 0. */ - *valuep = 0; - } - } - else - return 0; - } else if (*s == 'I' || *s == 'i') { - s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; - s++; if (s == send || (*s != 'F' && *s != 'f')) return 0; - s++; if (s < send && (*s == 'I' || *s == 'i')) { - s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; - s++; if (s == send || (*s != 'I' && *s != 'i')) return 0; - s++; if (s == send || (*s != 'T' && *s != 't')) return 0; - s++; if (s == send || (*s != 'Y' && *s != 'y')) return 0; - s++; - } - sawinf = 1; - } else if (*s == 'N' || *s == 'n') { - /* XXX TODO: There are signaling NaNs and quiet NaNs. */ - s++; if (s == send || (*s != 'A' && *s != 'a')) return 0; - s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; - s++; - sawnan = 1; - } else - return 0; - - if (sawinf) { - numtype &= IS_NUMBER_NEG; /* Keep track of sign */ - numtype |= IS_NUMBER_INFINITY | IS_NUMBER_NOT_INT; - } else if (sawnan) { - numtype &= IS_NUMBER_NEG; /* Keep track of sign */ - numtype |= IS_NUMBER_NAN | IS_NUMBER_NOT_INT; - } else if (s < send) { - /* we can have an optional exponent part */ - if (*s == 'e' || *s == 'E') { - /* The only flag we keep is sign. Blow away any "it's UV" */ - numtype &= IS_NUMBER_NEG; - numtype |= IS_NUMBER_NOT_INT; - s++; - if (s < send && (*s == '-' || *s == '+')) - s++; - if (s < send && isDIGIT(*s)) { - do { - s++; - } while (s < send && isDIGIT(*s)); - } - else - return 0; - } - } - while (s < send && isSPACE(*s)) - s++; - if (s >= send) - return numtype; - if (len == 10 && memEQ(pv, "0 but true", 10)) { - if (valuep) - *valuep = 0; - return IS_NUMBER_IN_UV; - } - return 0; -} -#endif -#endif - -/* - * The grok_* routines have been modified to use warn() instead of - * Perl_warner(). Also, 'hexdigit' was the former name of PL_hexdigit, - * which is why the stack variable has been renamed to 'xdigit'. - */ - -#ifndef grok_bin -#if defined(NEED_grok_bin) -static UV DPPP_(my_grok_bin)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); -static -#else -extern UV DPPP_(my_grok_bin)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); -#endif - -#ifdef grok_bin -# undef grok_bin -#endif -#define grok_bin(a,b,c,d) DPPP_(my_grok_bin)(aTHX_ a,b,c,d) -#define Perl_grok_bin DPPP_(my_grok_bin) - -#if defined(NEED_grok_bin) || defined(NEED_grok_bin_GLOBAL) -UV -DPPP_(my_grok_bin)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) -{ - const char *s = start; - STRLEN len = *len_p; - UV value = 0; - NV value_nv = 0; - - const UV max_div_2 = UV_MAX / 2; - bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; - bool overflowed = FALSE; - - if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) { - /* strip off leading b or 0b. - for compatibility silently suffer "b" and "0b" as valid binary - numbers. */ - if (len >= 1) { - if (s[0] == 'b') { - s++; - len--; - } - else if (len >= 2 && s[0] == '0' && s[1] == 'b') { - s+=2; - len-=2; - } - } - } - - for (; len-- && *s; s++) { - char bit = *s; - if (bit == '0' || bit == '1') { - /* Write it in this wonky order with a goto to attempt to get the - compiler to make the common case integer-only loop pretty tight. - With gcc seems to be much straighter code than old scan_bin. */ - redo: - if (!overflowed) { - if (value <= max_div_2) { - value = (value << 1) | (bit - '0'); - continue; - } - /* Bah. We're just overflowed. */ - warn("Integer overflow in binary number"); - overflowed = TRUE; - value_nv = (NV) value; - } - value_nv *= 2.0; - /* If an NV has not enough bits in its mantissa to - * represent a UV this summing of small low-order numbers - * is a waste of time (because the NV cannot preserve - * the low-order bits anyway): we could just remember when - * did we overflow and in the end just multiply value_nv by the - * right amount. */ - value_nv += (NV)(bit - '0'); - continue; - } - if (bit == '_' && len && allow_underscores && (bit = s[1]) - && (bit == '0' || bit == '1')) - { - --len; - ++s; - goto redo; - } - if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) - warn("Illegal binary digit '%c' ignored", *s); - break; - } - - if ( ( overflowed && value_nv > 4294967295.0) -#if UVSIZE > 4 - || (!overflowed && value > 0xffffffff ) -#endif - ) { - warn("Binary number > 0b11111111111111111111111111111111 non-portable"); - } - *len_p = s - start; - if (!overflowed) { - *flags = 0; - return value; - } - *flags = PERL_SCAN_GREATER_THAN_UV_MAX; - if (result) - *result = value_nv; - return UV_MAX; -} -#endif -#endif - -#ifndef grok_hex -#if defined(NEED_grok_hex) -static UV DPPP_(my_grok_hex)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); -static -#else -extern UV DPPP_(my_grok_hex)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); -#endif - -#ifdef grok_hex -# undef grok_hex -#endif -#define grok_hex(a,b,c,d) DPPP_(my_grok_hex)(aTHX_ a,b,c,d) -#define Perl_grok_hex DPPP_(my_grok_hex) - -#if defined(NEED_grok_hex) || defined(NEED_grok_hex_GLOBAL) -UV -DPPP_(my_grok_hex)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) -{ - const char *s = start; - STRLEN len = *len_p; - UV value = 0; - NV value_nv = 0; - - const UV max_div_16 = UV_MAX / 16; - bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; - bool overflowed = FALSE; - const char *xdigit; - - if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) { - /* strip off leading x or 0x. - for compatibility silently suffer "x" and "0x" as valid hex numbers. - */ - if (len >= 1) { - if (s[0] == 'x') { - s++; - len--; - } - else if (len >= 2 && s[0] == '0' && s[1] == 'x') { - s+=2; - len-=2; - } - } - } - - for (; len-- && *s; s++) { - xdigit = strchr((char *) PL_hexdigit, *s); - if (xdigit) { - /* Write it in this wonky order with a goto to attempt to get the - compiler to make the common case integer-only loop pretty tight. - With gcc seems to be much straighter code than old scan_hex. */ - redo: - if (!overflowed) { - if (value <= max_div_16) { - value = (value << 4) | ((xdigit - PL_hexdigit) & 15); - continue; - } - warn("Integer overflow in hexadecimal number"); - overflowed = TRUE; - value_nv = (NV) value; - } - value_nv *= 16.0; - /* If an NV has not enough bits in its mantissa to - * represent a UV this summing of small low-order numbers - * is a waste of time (because the NV cannot preserve - * the low-order bits anyway): we could just remember when - * did we overflow and in the end just multiply value_nv by the - * right amount of 16-tuples. */ - value_nv += (NV)((xdigit - PL_hexdigit) & 15); - continue; - } - if (*s == '_' && len && allow_underscores && s[1] - && (xdigit = strchr((char *) PL_hexdigit, s[1]))) - { - --len; - ++s; - goto redo; - } - if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) - warn("Illegal hexadecimal digit '%c' ignored", *s); - break; - } - - if ( ( overflowed && value_nv > 4294967295.0) -#if UVSIZE > 4 - || (!overflowed && value > 0xffffffff ) -#endif - ) { - warn("Hexadecimal number > 0xffffffff non-portable"); - } - *len_p = s - start; - if (!overflowed) { - *flags = 0; - return value; - } - *flags = PERL_SCAN_GREATER_THAN_UV_MAX; - if (result) - *result = value_nv; - return UV_MAX; -} -#endif -#endif - -#ifndef grok_oct -#if defined(NEED_grok_oct) -static UV DPPP_(my_grok_oct)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); -static -#else -extern UV DPPP_(my_grok_oct)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); -#endif - -#ifdef grok_oct -# undef grok_oct -#endif -#define grok_oct(a,b,c,d) DPPP_(my_grok_oct)(aTHX_ a,b,c,d) -#define Perl_grok_oct DPPP_(my_grok_oct) - -#if defined(NEED_grok_oct) || defined(NEED_grok_oct_GLOBAL) -UV -DPPP_(my_grok_oct)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) -{ - const char *s = start; - STRLEN len = *len_p; - UV value = 0; - NV value_nv = 0; - - const UV max_div_8 = UV_MAX / 8; - bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; - bool overflowed = FALSE; - - for (; len-- && *s; s++) { - /* gcc 2.95 optimiser not smart enough to figure that this subtraction - out front allows slicker code. */ - int digit = *s - '0'; - if (digit >= 0 && digit <= 7) { - /* Write it in this wonky order with a goto to attempt to get the - compiler to make the common case integer-only loop pretty tight. - */ - redo: - if (!overflowed) { - if (value <= max_div_8) { - value = (value << 3) | digit; - continue; - } - /* Bah. We're just overflowed. */ - warn("Integer overflow in octal number"); - overflowed = TRUE; - value_nv = (NV) value; - } - value_nv *= 8.0; - /* If an NV has not enough bits in its mantissa to - * represent a UV this summing of small low-order numbers - * is a waste of time (because the NV cannot preserve - * the low-order bits anyway): we could just remember when - * did we overflow and in the end just multiply value_nv by the - * right amount of 8-tuples. */ - value_nv += (NV)digit; - continue; - } - if (digit == ('_' - '0') && len && allow_underscores - && (digit = s[1] - '0') && (digit >= 0 && digit <= 7)) - { - --len; - ++s; - goto redo; - } - /* Allow \octal to work the DWIM way (that is, stop scanning - * as soon as non-octal characters are seen, complain only iff - * someone seems to want to use the digits eight and nine). */ - if (digit == 8 || digit == 9) { - if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) - warn("Illegal octal digit '%c' ignored", *s); - } - break; - } - - if ( ( overflowed && value_nv > 4294967295.0) -#if UVSIZE > 4 - || (!overflowed && value > 0xffffffff ) -#endif - ) { - warn("Octal number > 037777777777 non-portable"); - } - *len_p = s - start; - if (!overflowed) { - *flags = 0; - return value; - } - *flags = PERL_SCAN_GREATER_THAN_UV_MAX; - if (result) - *result = value_nv; - return UV_MAX; -} -#endif -#endif - -#if !defined(my_snprintf) -#if defined(NEED_my_snprintf) -static int DPPP_(my_my_snprintf)(char * buffer, const Size_t len, const char * format, ...); -static -#else -extern int DPPP_(my_my_snprintf)(char * buffer, const Size_t len, const char * format, ...); -#endif - -#define my_snprintf DPPP_(my_my_snprintf) -#define Perl_my_snprintf DPPP_(my_my_snprintf) - -#if defined(NEED_my_snprintf) || defined(NEED_my_snprintf_GLOBAL) - -int -DPPP_(my_my_snprintf)(char *buffer, const Size_t len, const char *format, ...) -{ - dTHX; - int retval; - va_list ap; - va_start(ap, format); -#ifdef HAS_VSNPRINTF - retval = vsnprintf(buffer, len, format, ap); -#else - retval = vsprintf(buffer, format, ap); -#endif - va_end(ap); - if (retval < 0 || (len > 0 && (Size_t)retval >= len)) - Perl_croak(aTHX_ "panic: my_snprintf buffer overflow"); - return retval; -} - -#endif -#endif - -#if !defined(my_sprintf) -#if defined(NEED_my_sprintf) -static int DPPP_(my_my_sprintf)(char * buffer, const char * pat, ...); -static -#else -extern int DPPP_(my_my_sprintf)(char * buffer, const char * pat, ...); -#endif - -#define my_sprintf DPPP_(my_my_sprintf) -#define Perl_my_sprintf DPPP_(my_my_sprintf) - -#if defined(NEED_my_sprintf) || defined(NEED_my_sprintf_GLOBAL) - -int -DPPP_(my_my_sprintf)(char *buffer, const char* pat, ...) -{ - va_list args; - va_start(args, pat); - vsprintf(buffer, pat, args); - va_end(args); - return strlen(buffer); -} - -#endif -#endif - -#ifdef NO_XSLOCKS -# ifdef dJMPENV -# define dXCPT dJMPENV; int rEtV = 0 -# define XCPT_TRY_START JMPENV_PUSH(rEtV); if (rEtV == 0) -# define XCPT_TRY_END JMPENV_POP; -# define XCPT_CATCH if (rEtV != 0) -# define XCPT_RETHROW JMPENV_JUMP(rEtV) -# else -# define dXCPT Sigjmp_buf oldTOP; int rEtV = 0 -# define XCPT_TRY_START Copy(top_env, oldTOP, 1, Sigjmp_buf); rEtV = Sigsetjmp(top_env, 1); if (rEtV == 0) -# define XCPT_TRY_END Copy(oldTOP, top_env, 1, Sigjmp_buf); -# define XCPT_CATCH if (rEtV != 0) -# define XCPT_RETHROW Siglongjmp(top_env, rEtV) -# endif -#endif - -#if !defined(my_strlcat) -#if defined(NEED_my_strlcat) -static Size_t DPPP_(my_my_strlcat)(char * dst, const char * src, Size_t size); -static -#else -extern Size_t DPPP_(my_my_strlcat)(char * dst, const char * src, Size_t size); -#endif - -#define my_strlcat DPPP_(my_my_strlcat) -#define Perl_my_strlcat DPPP_(my_my_strlcat) - -#if defined(NEED_my_strlcat) || defined(NEED_my_strlcat_GLOBAL) - -Size_t -DPPP_(my_my_strlcat)(char *dst, const char *src, Size_t size) -{ - Size_t used, length, copy; - - used = strlen(dst); - length = strlen(src); - if (size > 0 && used < size - 1) { - copy = (length >= size - used) ? size - used - 1 : length; - memcpy(dst + used, src, copy); - dst[used + copy] = '\0'; - } - return used + length; -} -#endif -#endif - -#if !defined(my_strlcpy) -#if defined(NEED_my_strlcpy) -static Size_t DPPP_(my_my_strlcpy)(char * dst, const char * src, Size_t size); -static -#else -extern Size_t DPPP_(my_my_strlcpy)(char * dst, const char * src, Size_t size); -#endif - -#define my_strlcpy DPPP_(my_my_strlcpy) -#define Perl_my_strlcpy DPPP_(my_my_strlcpy) - -#if defined(NEED_my_strlcpy) || defined(NEED_my_strlcpy_GLOBAL) - -Size_t -DPPP_(my_my_strlcpy)(char *dst, const char *src, Size_t size) -{ - Size_t length, copy; - - length = strlen(src); - if (size > 0) { - copy = (length >= size) ? size - 1 : length; - memcpy(dst, src, copy); - dst[copy] = '\0'; - } - return length; -} - -#endif -#endif -#ifndef PERL_PV_ESCAPE_QUOTE -# define PERL_PV_ESCAPE_QUOTE 0x0001 -#endif - -#ifndef PERL_PV_PRETTY_QUOTE -# define PERL_PV_PRETTY_QUOTE PERL_PV_ESCAPE_QUOTE -#endif - -#ifndef PERL_PV_PRETTY_ELLIPSES -# define PERL_PV_PRETTY_ELLIPSES 0x0002 -#endif - -#ifndef PERL_PV_PRETTY_LTGT -# define PERL_PV_PRETTY_LTGT 0x0004 -#endif - -#ifndef PERL_PV_ESCAPE_FIRSTCHAR -# define PERL_PV_ESCAPE_FIRSTCHAR 0x0008 -#endif - -#ifndef PERL_PV_ESCAPE_UNI -# define PERL_PV_ESCAPE_UNI 0x0100 -#endif - -#ifndef PERL_PV_ESCAPE_UNI_DETECT -# define PERL_PV_ESCAPE_UNI_DETECT 0x0200 -#endif - -#ifndef PERL_PV_ESCAPE_ALL -# define PERL_PV_ESCAPE_ALL 0x1000 -#endif - -#ifndef PERL_PV_ESCAPE_NOBACKSLASH -# define PERL_PV_ESCAPE_NOBACKSLASH 0x2000 -#endif - -#ifndef PERL_PV_ESCAPE_NOCLEAR -# define PERL_PV_ESCAPE_NOCLEAR 0x4000 -#endif - -#ifndef PERL_PV_ESCAPE_RE -# define PERL_PV_ESCAPE_RE 0x8000 -#endif - -#ifndef PERL_PV_PRETTY_NOCLEAR -# define PERL_PV_PRETTY_NOCLEAR PERL_PV_ESCAPE_NOCLEAR -#endif -#ifndef PERL_PV_PRETTY_DUMP -# define PERL_PV_PRETTY_DUMP PERL_PV_PRETTY_ELLIPSES|PERL_PV_PRETTY_QUOTE -#endif - -#ifndef PERL_PV_PRETTY_REGPROP -# define PERL_PV_PRETTY_REGPROP PERL_PV_PRETTY_ELLIPSES|PERL_PV_PRETTY_LTGT|PERL_PV_ESCAPE_RE -#endif - -/* Hint: pv_escape - * Note that unicode functionality is only backported to - * those perl versions that support it. For older perl - * versions, the implementation will fall back to bytes. - */ - -#ifndef pv_escape -#if defined(NEED_pv_escape) -static char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags); -static -#else -extern char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags); -#endif - -#ifdef pv_escape -# undef pv_escape -#endif -#define pv_escape(a,b,c,d,e,f) DPPP_(my_pv_escape)(aTHX_ a,b,c,d,e,f) -#define Perl_pv_escape DPPP_(my_pv_escape) - -#if defined(NEED_pv_escape) || defined(NEED_pv_escape_GLOBAL) - -char * -DPPP_(my_pv_escape)(pTHX_ SV *dsv, char const * const str, - const STRLEN count, const STRLEN max, - STRLEN * const escaped, const U32 flags) -{ - const char esc = flags & PERL_PV_ESCAPE_RE ? '%' : '\\'; - const char dq = flags & PERL_PV_ESCAPE_QUOTE ? '"' : esc; - char octbuf[32] = "%123456789ABCDF"; - STRLEN wrote = 0; - STRLEN chsize = 0; - STRLEN readsize = 1; -#if defined(is_utf8_string) && defined(utf8_to_uvchr) - bool isuni = flags & PERL_PV_ESCAPE_UNI ? 1 : 0; -#endif - const char *pv = str; - const char * const end = pv + count; - octbuf[0] = esc; - - if (!(flags & PERL_PV_ESCAPE_NOCLEAR)) - sv_setpvs(dsv, ""); - -#if defined(is_utf8_string) && defined(utf8_to_uvchr) - if ((flags & PERL_PV_ESCAPE_UNI_DETECT) && is_utf8_string((U8*)pv, count)) - isuni = 1; -#endif - - for (; pv < end && (!max || wrote < max) ; pv += readsize) { - const UV u = -#if defined(is_utf8_string) && defined(utf8_to_uvchr) - isuni ? utf8_to_uvchr((U8*)pv, &readsize) : -#endif - (U8)*pv; - const U8 c = (U8)u & 0xFF; - - if (u > 255 || (flags & PERL_PV_ESCAPE_ALL)) { - if (flags & PERL_PV_ESCAPE_FIRSTCHAR) - chsize = my_snprintf(octbuf, sizeof octbuf, - "%"UVxf, u); - else - chsize = my_snprintf(octbuf, sizeof octbuf, - "%cx{%"UVxf"}", esc, u); - } else if (flags & PERL_PV_ESCAPE_NOBACKSLASH) { - chsize = 1; - } else { - if (c == dq || c == esc || !isPRINT(c)) { - chsize = 2; - switch (c) { - case '\\' : /* fallthrough */ - case '%' : if (c == esc) - octbuf[1] = esc; - else - chsize = 1; - break; - case '\v' : octbuf[1] = 'v'; break; - case '\t' : octbuf[1] = 't'; break; - case '\r' : octbuf[1] = 'r'; break; - case '\n' : octbuf[1] = 'n'; break; - case '\f' : octbuf[1] = 'f'; break; - case '"' : if (dq == '"') - octbuf[1] = '"'; - else - chsize = 1; - break; - default: chsize = my_snprintf(octbuf, sizeof octbuf, - pv < end && isDIGIT((U8)*(pv+readsize)) - ? "%c%03o" : "%c%o", esc, c); - } - } else { - chsize = 1; - } - } - if (max && wrote + chsize > max) { - break; - } else if (chsize > 1) { - sv_catpvn(dsv, octbuf, chsize); - wrote += chsize; - } else { - char tmp[2]; - my_snprintf(tmp, sizeof tmp, "%c", c); - sv_catpvn(dsv, tmp, 1); - wrote++; - } - if (flags & PERL_PV_ESCAPE_FIRSTCHAR) - break; - } - if (escaped != NULL) - *escaped= pv - str; - return SvPVX(dsv); -} - -#endif -#endif - -#ifndef pv_pretty -#if defined(NEED_pv_pretty) -static char * DPPP_(my_pv_pretty)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, char const * const start_color, char const * const end_color, const U32 flags); -static -#else -extern char * DPPP_(my_pv_pretty)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, char const * const start_color, char const * const end_color, const U32 flags); -#endif - -#ifdef pv_pretty -# undef pv_pretty -#endif -#define pv_pretty(a,b,c,d,e,f,g) DPPP_(my_pv_pretty)(aTHX_ a,b,c,d,e,f,g) -#define Perl_pv_pretty DPPP_(my_pv_pretty) - -#if defined(NEED_pv_pretty) || defined(NEED_pv_pretty_GLOBAL) - -char * -DPPP_(my_pv_pretty)(pTHX_ SV *dsv, char const * const str, const STRLEN count, - const STRLEN max, char const * const start_color, char const * const end_color, - const U32 flags) -{ - const U8 dq = (flags & PERL_PV_PRETTY_QUOTE) ? '"' : '%'; - STRLEN escaped; - - if (!(flags & PERL_PV_PRETTY_NOCLEAR)) - sv_setpvs(dsv, ""); - - if (dq == '"') - sv_catpvs(dsv, "\""); - else if (flags & PERL_PV_PRETTY_LTGT) - sv_catpvs(dsv, "<"); - - if (start_color != NULL) - sv_catpv(dsv, D_PPP_CONSTPV_ARG(start_color)); - - pv_escape(dsv, str, count, max, &escaped, flags | PERL_PV_ESCAPE_NOCLEAR); - - if (end_color != NULL) - sv_catpv(dsv, D_PPP_CONSTPV_ARG(end_color)); - - if (dq == '"') - sv_catpvs(dsv, "\""); - else if (flags & PERL_PV_PRETTY_LTGT) - sv_catpvs(dsv, ">"); - - if ((flags & PERL_PV_PRETTY_ELLIPSES) && escaped < count) - sv_catpvs(dsv, "..."); - - return SvPVX(dsv); -} - -#endif -#endif - -#ifndef pv_display -#if defined(NEED_pv_display) -static char * DPPP_(my_pv_display)(pTHX_ SV * dsv, const char * pv, STRLEN cur, STRLEN len, STRLEN pvlim); -static -#else -extern char * DPPP_(my_pv_display)(pTHX_ SV * dsv, const char * pv, STRLEN cur, STRLEN len, STRLEN pvlim); -#endif - -#ifdef pv_display -# undef pv_display -#endif -#define pv_display(a,b,c,d,e) DPPP_(my_pv_display)(aTHX_ a,b,c,d,e) -#define Perl_pv_display DPPP_(my_pv_display) - -#if defined(NEED_pv_display) || defined(NEED_pv_display_GLOBAL) - -char * -DPPP_(my_pv_display)(pTHX_ SV *dsv, const char *pv, STRLEN cur, STRLEN len, STRLEN pvlim) -{ - pv_pretty(dsv, pv, cur, pvlim, NULL, NULL, PERL_PV_PRETTY_DUMP); - if (len > cur && pv[cur] == '\0') - sv_catpvs(dsv, "\\0"); - return SvPVX(dsv); -} - -#endif -#endif - -#endif /* _P_P_PORTABILITY_H_ */ - -/* End of File ppport.h */ From da9b75f5533dd83efb8ba3e1fb388daf374ab18d Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 30 Jul 2009 16:52:13 +0900 Subject: [PATCH 0269/1172] Checking in changes prior to tagging of version 0.08. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 4e7f1d7..c6e370c 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.08 + + - fixed PVNV issue... + 0.07 - do not use switch (SvTYPE(val)). --- perl/Changes | 4 ++++ perl/MANIFEST.SKIP | 1 + perl/lib/Data/MessagePack.pm | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 4e7f1d7..c6e370c 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.08 + + - fixed PVNV issue... + 0.07 - do not use switch (SvTYPE(val)). diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP index 1b268ef..f524216 100644 --- a/perl/MANIFEST.SKIP +++ b/perl/MANIFEST.SKIP @@ -20,3 +20,4 @@ ^\.git/ \.sw[pon]$ ^\.gitignore$ +ppport.h diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 38431eb..2ab33f1 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.07'; +our $VERSION = '0.08'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From aaef612a057908bc4227154ba15aee2bb66f642f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 6 Aug 2009 13:20:43 +0900 Subject: [PATCH 0270/1172] Makefile.am: don't package perl and ruby files --- Makefile.am | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/Makefile.am b/Makefile.am index 9e36092..3144972 100644 --- a/Makefile.am +++ b/Makefile.am @@ -11,38 +11,3 @@ nobase_include_HEADERS = \ msgpack/unpack_define.h \ msgpack/unpack_template.h -EXTRA_DIST = \ - ruby/bench.rb \ - ruby/extconf.rb \ - ruby/gem/README \ - ruby/gem/Rakefile \ - ruby/gem/test/test_helper.rb \ - ruby/gengem.sh \ - ruby/msgpack.gemspec \ - ruby/pack.c \ - ruby/pack.h \ - ruby/rbinit.c \ - ruby/test_case.rb \ - ruby/test_format.rb \ - ruby/test_pack.rb \ - ruby/unpack.c \ - ruby/unpack.h \ - perl/Makefile.PL \ - perl/MessagePack.c \ - perl/benchmark/deserialize.pl \ - perl/benchmark/serialize.pl \ - perl/lib/Data/MessagePack.pm \ - perl/lib/Data/MessagePack \ - perl/lib/Data/MessagePack/Unpacker.pod \ - perl/pack.c \ - perl/ppport.h \ - perl/t/00_compile.t \ - perl/t/01_pack.t \ - perl/t/02_unpack.t \ - perl/t/03_stream_unpack.t \ - perl/t/04_invert.t \ - perl/t/Util.pm \ - perl/t/data.pl \ - perl/unpack.c \ - perl/xt/99_pod.t - From f7a9805f7b3b6413c307863f1f16b320abb665ec Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 6 Aug 2009 13:26:34 +0900 Subject: [PATCH 0271/1172] operator>> (object, std::vector): return reference; don't copy --- cpp/type/array.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp index 6027251..5b80dc1 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/array.hpp @@ -25,7 +25,7 @@ namespace msgpack { template -inline std::vector operator>> (object o, std::vector& v) +inline std::vector& operator>> (object o, std::vector& v) { if(o.type != type::ARRAY) { throw type_error(); } v.resize(o.via.array.size); From ec8932d6a11977136516fb2b7a3e3015cab0c3ac Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 6 Aug 2009 13:28:41 +0900 Subject: [PATCH 0272/1172] fix missing tuple<>::value_type --- cpp/type/tuple.hpp.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index 586d84c..0105feb 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -103,6 +103,7 @@ tuple, A<%=j%><%}%>> make_tuple(typename tuple_type::tr template <> struct tuple<> { + typedef tuple<> value_type; }; <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> From 3afa9f265ebb283f26cb516ceca52cc083916cf7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 6 Aug 2009 13:51:49 +0900 Subject: [PATCH 0273/1172] cpp: add msgpack::type::define --- configure.in | 2 +- cpp/Makefile.am | 7 ++- cpp/type.hpp | 1 + cpp/type/define.hpp.erb | 102 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 cpp/type/define.hpp.erb diff --git a/configure.in b/configure.in index 45a04a9..8646d1e 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.3) +AM_INIT_AUTOMAKE(msgpack, 0.3.4) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index b9126f8..5efac3f 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -19,7 +19,8 @@ nobase_include_HEADERS = \ msgpack/type/map.hpp \ msgpack/type/nil.hpp \ msgpack/type/raw.hpp \ - msgpack/type/tuple.hpp + msgpack/type/tuple.hpp \ + msgpack/type/define.hpp # FIXME object.lo: msgpack/type/tuple.hpp msgpack/zone.hpp @@ -28,6 +29,10 @@ msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb $(ERB) $< > $@.tmp mv $@.tmp $@ +msgpack/type/define.hpp: msgpack/type/define.hpp.erb + $(ERB) $< > $@.tmp + mv $@.tmp $@ + msgpack/zone.hpp: msgpack/zone.hpp.erb $(ERB) $< > $@.tmp mv $@.tmp $@ diff --git a/cpp/type.hpp b/cpp/type.hpp index 1dfd414..8ffe3bf 100644 --- a/cpp/type.hpp +++ b/cpp/type.hpp @@ -6,4 +6,5 @@ #include "msgpack/type/nil.hpp" #include "msgpack/type/raw.hpp" #include "msgpack/type/tuple.hpp" +#include "msgpack/type/define.hpp" diff --git a/cpp/type/define.hpp.erb b/cpp/type/define.hpp.erb new file mode 100644 index 0000000..9be4f77 --- /dev/null +++ b/cpp/type/define.hpp.erb @@ -0,0 +1,102 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_DEFINE_HPP__ +#define MSGPACK_TYPE_DEFINE_HPP__ + +#define MSGPACK_DEFINE(...) \ + template \ + void msgpack_pack(Packer& pk) const \ + { \ + msgpack::type::make_define(__VA_ARGS__).msgpack_pack(pk); \ + } \ + \ + void msgpack_unpack(msgpack::object o) \ + { \ + msgpack::type::make_define(__VA_ARGS__).msgpack_unpack(o); \ + } + +namespace msgpack { +namespace type { + + +<% GENERATION_LIMIT = 15 %> +template , typename A<%=i%> = void<%}%>> +struct define; + + +template <> +struct define<> { + typedef define<> value_type; + typedef tuple<> tuple_type; + template + void msgpack_pack(Packer& pk) const + { + pk.pack_array(1); + } + void msgpack_unpack(msgpack::object o) + { + if(o.type != type::ARRAY) { throw type_error(); } + } +}; + +<%0.upto(GENERATION_LIMIT) {|i|%> +template , typename A<%=j%><%}%>> +struct define, A<%=j%><%}%>> { + typedef define, A<%=j%><%}%>> value_type; + typedef tuple, A<%=j%><%}%>> tuple_type; + define(A0& _a0<%1.upto(i) {|j|%>, A<%=j%>& _a<%=j%><%}%>) : + a0(_a0)<%1.upto(i) {|j|%>, a<%=j%>(_a<%=j%>)<%}%> {} + template + void msgpack_pack(Packer& pk) const + { + pk.pack_array(<%=i+1%>); + <%0.upto(i) {|j|%> + pk.pack(a<%=j%>);<%}%> + } + void msgpack_unpack(msgpack::object o) + { + if(o.type != type::ARRAY) { throw type_error(); } + const size_t size = o.via.array.size; + <%0.upto(i) {|j|%> + if(size <= <%=j%>) { return; } o.via.array.ptr[<%=j%>].convert(&a<%=j%>);<%}%> + } + <%0.upto(i) {|j|%> + A<%=j%>& a<%=j%>;<%}%> +}; +<%}%> + +define<> make_define() +{ + return define<>(); +} + +<%0.upto(GENERATION_LIMIT) {|i|%> +template , typename A<%=j%><%}%>> +define, A<%=j%><%}%>> make_define(A0& a0<%1.upto(i) {|j|%>, A<%=j%>& a<%=j%><%}%>) +{ + return define, A<%=j%><%}%>>(a0<%1.upto(i) {|j|%>, a<%=j%><%}%>); +} +<%}%> + + +} // namespace type +} // namespace msgpack + + +#endif /* msgpack/type/define.hpp */ + From 7fbe845434937ac6dcf5b89e0d80661443eafd90 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 6 Aug 2009 13:59:48 +0900 Subject: [PATCH 0274/1172] cpp: msgpack::define is obsolete --- cpp/object.hpp | 1 + cpp/type/define.hpp.erb | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 07917c2..9ee575f 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -129,6 +129,7 @@ private: }; +// obsolete template class define : public Type { public: diff --git a/cpp/type/define.hpp.erb b/cpp/type/define.hpp.erb index 9be4f77..c13aa16 100644 --- a/cpp/type/define.hpp.erb +++ b/cpp/type/define.hpp.erb @@ -24,7 +24,6 @@ { \ msgpack::type::make_define(__VA_ARGS__).msgpack_pack(pk); \ } \ - \ void msgpack_unpack(msgpack::object o) \ { \ msgpack::type::make_define(__VA_ARGS__).msgpack_unpack(o); \ From 1375732c805264cf43baa112a5a45e4c0c43b8e6 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 7 Aug 2009 11:15:33 +0900 Subject: [PATCH 0275/1172] add --- example/custom.cc | 58 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 example/custom.cc diff --git a/example/custom.cc b/example/custom.cc new file mode 100644 index 0000000..835ebed --- /dev/null +++ b/example/custom.cc @@ -0,0 +1,58 @@ +#include +#include +#include + +class old_class { +public: + old_class() : value("default") { } + + std::string value; + + MSGPACK_DEFINE(value); +}; + +class new_class { +public: + new_class() : value("default"), flag(-1) { } + + std::string value; + int flag; + + MSGPACK_DEFINE(value, flag); +}; + +int main(void) +{ + { + old_class oc; + new_class nc; + + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, oc); + + msgpack::zone zone; + msgpack::object obj; + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &zone, &obj); + + obj.convert(&nc); + + std::cout << obj << " value=" << nc.value << " flag=" << nc.flag << std::endl; + } + + { + new_class nc; + old_class oc; + + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, nc); + + msgpack::zone zone; + msgpack::object obj; + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &zone, &obj); + + obj.convert(&oc); + + std::cout << obj << " value=" << oc.value << std::endl; + } +} + From b4cb5e23c0a235d61d4a63acf4624f96f20f620b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 7 Aug 2009 11:28:23 +0900 Subject: [PATCH 0276/1172] fix Makefile.am --- cpp/Makefile.am | 2 ++ example/stream.cc | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 5efac3f..f630832 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -39,10 +39,12 @@ msgpack/zone.hpp: msgpack/zone.hpp.erb MOSTLYCLEANFILES = \ msgpack/type/tuple.hpp \ + msgpack/type/define.hpp \ msgpack/zone.hpp EXTRA_DIST = \ msgpack/type/tuple.hpp.erb \ + msgpack/type/define.hpp.erb \ msgpack/zone.hpp.erb libmsgpack_la_LIBADD = -L../c -lmsgpackc diff --git a/example/stream.cc b/example/stream.cc index 1e2e733..2241935 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -22,14 +22,14 @@ public: ssize_t count = read(m_sock, m_pac.buffer(), m_pac.buffer_capacity()); - if(count < 0) { + if(count <= 0) { + if(count == 0) { + throw std::runtime_error("connection closed"); + } if(errno == EAGAIN || errno == EINTR) { return; - } else { - throw std::runtime_error(strerror(errno)); } - } else if(count == 0) { - throw std::runtime_error("connection closed"); + throw std::runtime_error(strerror(errno)); } m_pac.buffer_consumed(count); From be32e3b1fb3fc45747c8df13f9a2eed5626e7c47 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 7 Aug 2009 13:31:42 +0900 Subject: [PATCH 0277/1172] cpp: add missing msgpack::type::make_tuple() --- cpp/type/tuple.hpp.erb | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index 0105feb..d8ddbcd 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -64,7 +64,6 @@ struct tuple_type { typedef const T& transparent_reference; }; - <%0.upto(GENERATION_LIMIT) {|i|%> <%0.upto(i) {|j|%> template , typename A<%=k%><%}%>> @@ -78,7 +77,6 @@ private: <%}%> <%}%> - <%0.upto(GENERATION_LIMIT) {|i|%> <%0.upto(i) {|j|%> template , typename A<%=k%><%}%>> @@ -91,16 +89,6 @@ private: <%}%> <%}%> - -<%0.upto(GENERATION_LIMIT) {|i|%> -template , typename A<%=j%><%}%>> -tuple, A<%=j%><%}%>> make_tuple(typename tuple_type::transparent_reference a0<%1.upto(i) {|j|%>, typename tuple_type>::transparent_reference a<%=j%><%}%>) -{ - return tuple, A<%=j%><%}%>>(a0<%1.upto(i) {|j|%>, a<%=j%><%}%>); -} -<%}%> - - template <> struct tuple<> { typedef tuple<> value_type; @@ -122,6 +110,18 @@ struct tuple, A<%=j%><%}%>> { }; <%}%> +inline tuple<> make_tuple() +{ + return tuple<>(); +} +<%0.upto(GENERATION_LIMIT) {|i|%> +template , typename A<%=j%><%}%>> +tuple, A<%=j%><%}%>> make_tuple(typename tuple_type::transparent_reference a0<%1.upto(i) {|j|%>, typename tuple_type>::transparent_reference a<%=j%><%}%>) +{ + return tuple, A<%=j%><%}%>>(a0<%1.upto(i) {|j|%>, a<%=j%><%}%>); +} +<%}%> + } // namespace type @@ -144,7 +144,6 @@ type::tuple, A<%=j%><%}%>>& operator>> ( } <%}%> - template const packer& operator<< ( packer& o, @@ -164,7 +163,6 @@ const packer& operator<< ( } <%}%> - } // namespace msgpack #endif /* msgpack/type/tuple.hpp */ From 95a6316cc7df5617090a948f347bdf2c33d45b79 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 7 Aug 2009 13:32:07 +0900 Subject: [PATCH 0278/1172] cpp: fix msgpack::type::make_define() --- cpp/type/define.hpp.erb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cpp/type/define.hpp.erb b/cpp/type/define.hpp.erb index c13aa16..d9d3fb1 100644 --- a/cpp/type/define.hpp.erb +++ b/cpp/type/define.hpp.erb @@ -52,7 +52,6 @@ struct define<> { if(o.type != type::ARRAY) { throw type_error(); } } }; - <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> struct define, A<%=j%><%}%>> { @@ -79,11 +78,10 @@ struct define, A<%=j%><%}%>> { }; <%}%> -define<> make_define() +inline define<> make_define() { return define<>(); } - <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> define, A<%=j%><%}%>> make_define(A0& a0<%1.upto(i) {|j|%>, A<%=j%>& a<%=j%><%}%>) @@ -92,7 +90,6 @@ define, A<%=j%><%}%>> make_define(A0& a0<%1.upto(i) {|j|%>, } <%}%> - } // namespace type } // namespace msgpack From a5705183d6dc2c64ca01716bb5b06d574e4c2a5f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 7 Aug 2009 13:32:32 +0900 Subject: [PATCH 0279/1172] 0.3.5 --- configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 8646d1e..202ca0d 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.4) +AM_INIT_AUTOMAKE(msgpack, 0.3.5) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) From 0491768fb2a9b2ba0c52de838253981ed90111cd Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 10 Aug 2009 15:58:43 +0900 Subject: [PATCH 0280/1172] c++: fix Makefile.am --- cpp/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index f630832..b7bf069 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -23,7 +23,7 @@ nobase_include_HEADERS = \ msgpack/type/define.hpp # FIXME -object.lo: msgpack/type/tuple.hpp msgpack/zone.hpp +object.lo: msgpack/type/tuple.hpp msgpack/type/define.hpp msgpack/zone.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb $(ERB) $< > $@.tmp From 394331cd4ece745b9c5b5eb155560aaece8dc2bc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 10 Aug 2009 17:34:21 +0900 Subject: [PATCH 0281/1172] c++: add std::pair and std::set serializer --- cpp/Makefile.am | 2 ++ cpp/type.hpp | 2 ++ cpp/type/pair.hpp | 51 ++++++++++++++++++++++++++++++++++++++++++ cpp/type/set.hpp | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+) create mode 100644 cpp/type/pair.hpp create mode 100644 cpp/type/set.hpp diff --git a/cpp/Makefile.am b/cpp/Makefile.am index b7bf069..edbeb28 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -18,7 +18,9 @@ nobase_include_HEADERS = \ msgpack/type/integer.hpp \ msgpack/type/map.hpp \ msgpack/type/nil.hpp \ + msgpack/type/pair.hpp \ msgpack/type/raw.hpp \ + msgpack/type/set.hpp \ msgpack/type/tuple.hpp \ msgpack/type/define.hpp diff --git a/cpp/type.hpp b/cpp/type.hpp index 8ffe3bf..2a6aa62 100644 --- a/cpp/type.hpp +++ b/cpp/type.hpp @@ -4,7 +4,9 @@ #include "msgpack/type/integer.hpp" #include "msgpack/type/map.hpp" #include "msgpack/type/nil.hpp" +#include "msgpack/type/pair.hpp" #include "msgpack/type/raw.hpp" +#include "msgpack/type/set.hpp" #include "msgpack/type/tuple.hpp" #include "msgpack/type/define.hpp" diff --git a/cpp/type/pair.hpp b/cpp/type/pair.hpp new file mode 100644 index 0000000..316b9fe --- /dev/null +++ b/cpp/type/pair.hpp @@ -0,0 +1,51 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_PAIR_HPP__ +#define MSGPACK_TYPE_PAIR_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { + + +template +inline std::pair& operator>> (object o, std::pair& v) +{ + if(o.type != type::ARRAY) { throw type_error(); } + if(o.via.array.size != 2) { throw type_error(); } + o.via.array.ptr[0].convert(&v.first); + o.via.array.ptr[1].convert(&v.second); + return v; +} + + +template +inline packer& operator<< (packer& o, const std::pair& v) +{ + o.pack_array(2); + o.pack(v.first); + o.pack(v.second); + return o; +} + + +} // namespace msgpack + +#endif /* msgpack/type/pair.hpp */ + diff --git a/cpp/type/set.hpp b/cpp/type/set.hpp new file mode 100644 index 0000000..3f1920a --- /dev/null +++ b/cpp/type/set.hpp @@ -0,0 +1,56 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_SET_HPP__ +#define MSGPACK_TYPE_SET_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { + + +template +inline std::set& operator>> (object o, std::set& v) +{ + if(o.type != type::ARRAY) { throw type_error(); } + object* p = o.via.array.ptr + o.via.array.size; + object* const pbegin = o.via.array.ptr; + while(p > pbegin) { + --p; + v.insert(p->as()); + } + return v; +} + + +template +inline packer& operator<< (packer& o, const std::set& v) +{ + o.pack_array(v.size()); + for(typename std::set::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(*it); + } + return o; +} + + +} // namespace msgpack + +#endif /* msgpack/type/set.hpp */ + From 0627324da62922d332c75ec51525141329187beb Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 10 Aug 2009 18:26:01 +0900 Subject: [PATCH 0282/1172] c++: rebuild type/*.hpp --- cpp/Makefile.am | 8 +++-- cpp/type.hpp | 8 +++-- cpp/type/{boolean.hpp => bool.hpp} | 4 +-- cpp/type/deque.hpp | 56 ++++++++++++++++++++++++++++++ cpp/type/float.hpp | 2 -- cpp/type/{integer.hpp => int.hpp} | 6 ++-- cpp/type/list.hpp | 56 ++++++++++++++++++++++++++++++ cpp/type/pair.hpp | 1 - cpp/type/raw.hpp | 19 ---------- cpp/type/string.hpp | 47 +++++++++++++++++++++++++ cpp/type/{array.hpp => vector.hpp} | 7 ++-- 11 files changed, 177 insertions(+), 37 deletions(-) rename cpp/type/{boolean.hpp => bool.hpp} (94%) create mode 100644 cpp/type/deque.hpp rename cpp/type/{integer.hpp => int.hpp} (97%) create mode 100644 cpp/type/list.hpp create mode 100644 cpp/type/string.hpp rename cpp/type/{array.hpp => vector.hpp} (93%) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index edbeb28..45cc13c 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -12,15 +12,17 @@ nobase_include_HEADERS = \ msgpack/object.hpp \ msgpack/zone.hpp \ msgpack/type.hpp \ - msgpack/type/array.hpp \ - msgpack/type/boolean.hpp \ + msgpack/type/bool.hpp \ msgpack/type/float.hpp \ - msgpack/type/integer.hpp \ + msgpack/type/int.hpp \ + msgpack/type/list.hpp \ msgpack/type/map.hpp \ msgpack/type/nil.hpp \ msgpack/type/pair.hpp \ msgpack/type/raw.hpp \ msgpack/type/set.hpp \ + msgpack/type/string.hpp \ + msgpack/type/vector.hpp \ msgpack/type/tuple.hpp \ msgpack/type/define.hpp diff --git a/cpp/type.hpp b/cpp/type.hpp index 2a6aa62..2bd805d 100644 --- a/cpp/type.hpp +++ b/cpp/type.hpp @@ -1,12 +1,14 @@ -#include "msgpack/type/array.hpp" -#include "msgpack/type/boolean.hpp" +#include "msgpack/type/bool.hpp" #include "msgpack/type/float.hpp" -#include "msgpack/type/integer.hpp" +#include "msgpack/type/int.hpp" +#include "msgpack/type/list.hpp" #include "msgpack/type/map.hpp" #include "msgpack/type/nil.hpp" #include "msgpack/type/pair.hpp" #include "msgpack/type/raw.hpp" #include "msgpack/type/set.hpp" +#include "msgpack/type/string.hpp" +#include "msgpack/type/vector.hpp" #include "msgpack/type/tuple.hpp" #include "msgpack/type/define.hpp" diff --git a/cpp/type/boolean.hpp b/cpp/type/bool.hpp similarity index 94% rename from cpp/type/boolean.hpp rename to cpp/type/bool.hpp index 86bd697..f3ac6fa 100644 --- a/cpp/type/boolean.hpp +++ b/cpp/type/bool.hpp @@ -15,8 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. // -#ifndef MSGPACK_TYPE_BOOLEAN_HPP__ -#define MSGPACK_TYPE_BOOLEAN_HPP__ +#ifndef MSGPACK_TYPE_BOOL_HPP__ +#define MSGPACK_TYPE_BOOL_HPP__ #include "msgpack/object.hpp" #include diff --git a/cpp/type/deque.hpp b/cpp/type/deque.hpp new file mode 100644 index 0000000..d34d243 --- /dev/null +++ b/cpp/type/deque.hpp @@ -0,0 +1,56 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_DEQUE_HPP__ +#define MSGPACK_TYPE_DEQUE_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { + + +template +inline std::deque& operator>> (object o, std::deque& v) +{ + if(o.type != type::ARRAY) { throw type_error(); } + v.resize(o.via.array.size); + object* p = o.via.array.ptr; + object* const pend = o.via.array.ptr + o.via.array.size; + typename std::deque::iterator it = v.begin(); + for(; p < pend; ++p, ++it) { + p->convert(&*it); + } + return v; +} + +template +inline packer& operator<< (packer& o, const std::deque& v) +{ + o.pack_array(v.size()); + for(typename std::deque::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(*it); + } + return o; +} + + +} // namespace msgpack + +#endif /* msgpack/type/deque.hpp */ + diff --git a/cpp/type/float.hpp b/cpp/type/float.hpp index 108709d..390e340 100644 --- a/cpp/type/float.hpp +++ b/cpp/type/float.hpp @@ -34,7 +34,6 @@ inline float& operator>> (object o, float& v) return v; } - template inline packer& operator<< (packer& o, const float& v) { @@ -50,7 +49,6 @@ inline double& operator>> (object o, double& v) return v; } - template inline packer& operator<< (packer& o, const double& v) { diff --git a/cpp/type/integer.hpp b/cpp/type/int.hpp similarity index 97% rename from cpp/type/integer.hpp rename to cpp/type/int.hpp index ecb7b89..8fdc386 100644 --- a/cpp/type/integer.hpp +++ b/cpp/type/int.hpp @@ -15,8 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. // -#ifndef MSGPACK_TYPE_INTEGER_HPP__ -#define MSGPACK_TYPE_INTEGER_HPP__ +#ifndef MSGPACK_TYPE_INT_HPP__ +#define MSGPACK_TYPE_INT_HPP__ #include "msgpack/object.hpp" #include @@ -143,5 +143,5 @@ inline packer& operator<< (packer& o, const unsigned long long& } // namespace msgpack -#endif /* msgpack/type/integer.hpp */ +#endif /* msgpack/type/int.hpp */ diff --git a/cpp/type/list.hpp b/cpp/type/list.hpp new file mode 100644 index 0000000..6ecc02f --- /dev/null +++ b/cpp/type/list.hpp @@ -0,0 +1,56 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_LIST_HPP__ +#define MSGPACK_TYPE_LIST_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { + + +template +inline std::list& operator>> (object o, std::list& v) +{ + if(o.type != type::ARRAY) { throw type_error(); } + v.resize(o.via.array.size); + object* p = o.via.array.ptr; + object* const pend = o.via.array.ptr + o.via.array.size; + typename std::list::iterator it = v.begin(); + for(; p < pend; ++p, ++it) { + p->convert(&*it); + } + return v; +} + +template +inline packer& operator<< (packer& o, const std::list& v) +{ + o.pack_array(v.size()); + for(typename std::list::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(*it); + } + return o; +} + + +} // namespace msgpack + +#endif /* msgpack/type/list.hpp */ + diff --git a/cpp/type/pair.hpp b/cpp/type/pair.hpp index 316b9fe..ba72c1f 100644 --- a/cpp/type/pair.hpp +++ b/cpp/type/pair.hpp @@ -34,7 +34,6 @@ inline std::pair& operator>> (object o, std::pair& v) return v; } - template inline packer& operator<< (packer& o, const std::pair& v) { diff --git a/cpp/type/raw.hpp b/cpp/type/raw.hpp index b6ace3f..8c68aba 100644 --- a/cpp/type/raw.hpp +++ b/cpp/type/raw.hpp @@ -69,16 +69,6 @@ inline type::raw_ref& operator>> (object o, type::raw_ref& v) return v; } - -inline std::string& operator>> (object o, std::string& v) -{ - type::raw_ref r; - o >> r; - v.assign(r.ptr, r.size); - return v; -} - - template inline packer& operator<< (packer& o, const type::raw_ref& v) { @@ -88,15 +78,6 @@ inline packer& operator<< (packer& o, const type::raw_ref& v) } -template -inline packer& operator<< (packer& o, const std::string& v) -{ - o.pack_raw(v.size()); - o.pack_raw_body(v.data(), v.size()); - return o; -} - - } // namespace msgpack #endif /* msgpack/type/raw.hpp */ diff --git a/cpp/type/string.hpp b/cpp/type/string.hpp new file mode 100644 index 0000000..2a23058 --- /dev/null +++ b/cpp/type/string.hpp @@ -0,0 +1,47 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_STRING_HPP__ +#define MSGPACK_TYPE_STRING_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { + + +inline std::string& operator>> (object o, std::string& v) +{ + type::raw_ref r; + o >> r; + v.assign(r.ptr, r.size); + return v; +} + +template +inline packer& operator<< (packer& o, const std::string& v) +{ + o.pack_raw(v.size()); + o.pack_raw_body(v.data(), v.size()); + return o; +} + + +} // namespace msgpack + +#endif /* msgpack/type/string.hpp */ + diff --git a/cpp/type/array.hpp b/cpp/type/vector.hpp similarity index 93% rename from cpp/type/array.hpp rename to cpp/type/vector.hpp index 5b80dc1..754cdc0 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/vector.hpp @@ -15,8 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. // -#ifndef MSGPACK_TYPE_ARRAY_HPP__ -#define MSGPACK_TYPE_ARRAY_HPP__ +#ifndef MSGPACK_TYPE_VECTOR_HPP__ +#define MSGPACK_TYPE_VECTOR_HPP__ #include "msgpack/object.hpp" #include @@ -38,7 +38,6 @@ inline std::vector& operator>> (object o, std::vector& v) return v; } - template inline packer& operator<< (packer& o, const std::vector& v) { @@ -53,5 +52,5 @@ inline packer& operator<< (packer& o, const std::vector& v) } // namespace msgpack -#endif /* msgpack/type/array.hpp */ +#endif /* msgpack/type/vector.hpp */ From a62a5d6c69837e474f9cc7260017de3fe61a92eb Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 10 Aug 2009 18:33:35 +0900 Subject: [PATCH 0283/1172] c++: fix type.hpp --- cpp/Makefile.am | 1 + cpp/type.hpp | 1 + 2 files changed, 2 insertions(+) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 45cc13c..f9d1fec 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -16,6 +16,7 @@ nobase_include_HEADERS = \ msgpack/type/float.hpp \ msgpack/type/int.hpp \ msgpack/type/list.hpp \ + msgpack/type/deque.hpp \ msgpack/type/map.hpp \ msgpack/type/nil.hpp \ msgpack/type/pair.hpp \ diff --git a/cpp/type.hpp b/cpp/type.hpp index 2bd805d..fafa674 100644 --- a/cpp/type.hpp +++ b/cpp/type.hpp @@ -2,6 +2,7 @@ #include "msgpack/type/float.hpp" #include "msgpack/type/int.hpp" #include "msgpack/type/list.hpp" +#include "msgpack/type/deque.hpp" #include "msgpack/type/map.hpp" #include "msgpack/type/nil.hpp" #include "msgpack/type/pair.hpp" From 92952f656691f5907e00c0a5ab45e3423d5429cb Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 10 Aug 2009 22:16:44 +0900 Subject: [PATCH 0284/1172] c++: fix object >> std::string --- configure.in | 2 +- cpp/type/bool.hpp | 1 - cpp/type/set.hpp | 1 - cpp/type/string.hpp | 5 ++--- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/configure.in b/configure.in index 202ca0d..7cdb99d 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.5) +AM_INIT_AUTOMAKE(msgpack, 0.3.6) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/cpp/type/bool.hpp b/cpp/type/bool.hpp index f3ac6fa..b945d85 100644 --- a/cpp/type/bool.hpp +++ b/cpp/type/bool.hpp @@ -31,7 +31,6 @@ inline bool& operator>> (object o, bool& v) return v; } - template inline packer& operator<< (packer& o, const bool& v) { diff --git a/cpp/type/set.hpp b/cpp/type/set.hpp index 3f1920a..11db2b3 100644 --- a/cpp/type/set.hpp +++ b/cpp/type/set.hpp @@ -37,7 +37,6 @@ inline std::set& operator>> (object o, std::set& v) return v; } - template inline packer& operator<< (packer& o, const std::set& v) { diff --git a/cpp/type/string.hpp b/cpp/type/string.hpp index 2a23058..a085d53 100644 --- a/cpp/type/string.hpp +++ b/cpp/type/string.hpp @@ -26,9 +26,8 @@ namespace msgpack { inline std::string& operator>> (object o, std::string& v) { - type::raw_ref r; - o >> r; - v.assign(r.ptr, r.size); + if(o.type != type::RAW) { throw type_error(); } + v.assign(o.via.raw.ptr, o.via.raw.size); return v; } From b2381d0513e3ec9a874643b7faf7afeb4490a1e7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 19 Aug 2009 15:10:10 +0900 Subject: [PATCH 0285/1172] cpp: fix overflow check for minimum number of signed integer type --- cpp/type/int.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/type/int.hpp b/cpp/type/int.hpp index 8fdc386..03500f7 100644 --- a/cpp/type/int.hpp +++ b/cpp/type/int.hpp @@ -37,7 +37,7 @@ namespace detail { { throw type_error(); } return o.via.u64; } else if(o.type == type::NEGATIVE_INTEGER) { - if(o.via.i64 < (int64_t)-std::numeric_limits::max()) + if(o.via.i64 < (int64_t)std::numeric_limits::min()) { throw type_error(); } return o.via.i64; } From 387eca6fbf8c18bf32f35eeb7dab53f2be670c33 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 19 Aug 2009 17:36:05 +0900 Subject: [PATCH 0286/1172] cpp-0.3.7 --- configure.in | 2 +- cpp/object.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/configure.in b/configure.in index 7cdb99d..c241ecf 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.6) +AM_INIT_AUTOMAKE(msgpack, 0.3.7) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/cpp/object.cpp b/cpp/object.cpp index eb45f77..a7adb71 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -16,7 +16,6 @@ // limitations under the License. // #include "msgpack/object.hpp" - #include namespace msgpack { From 3a39accb0b9e4fa6cebc1bafc5086366b5014778 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 19 Aug 2009 17:47:22 +0900 Subject: [PATCH 0287/1172] cpp: preprocess eruby templates on ./bootstrap; released package doesn't require erb --- Makefile.am | 1 - README | 4 +--- bootstrap | 5 +---- configure.in | 5 ----- cpp/Makefile.am | 25 ------------------------- 5 files changed, 2 insertions(+), 38 deletions(-) diff --git a/Makefile.am b/Makefile.am index 3144972..be3d75f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,5 +1,4 @@ if ENABLE_CXX -export ERB SUBDIRS = c cpp else SUBDIRS = c diff --git a/README b/README index 2a1a3e0..66ca276 100644 --- a/README +++ b/README @@ -8,9 +8,7 @@ Binary-based efficient data interchange format. MessagePack is only tested on Linux and Mac OS X, but it may run on other UNIX-like platforms. - Following programs are required to build: - - gcc >= 4.1 with C++ support - - ruby >= 1.8 (ruby is used as a preprocessor) + gcc >= 4.1 is required to build. *Installation diff --git a/bootstrap b/bootstrap index 9ddb3da..8ac504b 100755 --- a/bootstrap +++ b/bootstrap @@ -31,11 +31,8 @@ if test x"$1" = x"--help"; then fi -#if [ -z "$NO_NEST" ];then -# cd c && ./bootstrap $@; cd .. -# cd cpp && ./bootstrap $@; cd .. -#fi mkdir -p ac +(cd cpp && ./preprocess.sh $@; cd ..) ACLOCAL="aclocal" diff --git a/configure.in b/configure.in index c241ecf..de71fc9 100644 --- a/configure.in +++ b/configure.in @@ -23,11 +23,6 @@ if test "$enable_cxx" != "no"; then if test "" = "$CXXFLAGS"; then CXXFLAGS="-g -O4" fi - - AC_CHECK_PROG(ERB, erb, erb) - if test "x$ERB" = x; then - AC_MSG_ERROR([cannot find erb. Ruby is needed to build.]) - fi fi # FIXME diff --git a/cpp/Makefile.am b/cpp/Makefile.am index f9d1fec..2c11535 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -27,31 +27,6 @@ nobase_include_HEADERS = \ msgpack/type/tuple.hpp \ msgpack/type/define.hpp -# FIXME -object.lo: msgpack/type/tuple.hpp msgpack/type/define.hpp msgpack/zone.hpp - -msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb - $(ERB) $< > $@.tmp - mv $@.tmp $@ - -msgpack/type/define.hpp: msgpack/type/define.hpp.erb - $(ERB) $< > $@.tmp - mv $@.tmp $@ - -msgpack/zone.hpp: msgpack/zone.hpp.erb - $(ERB) $< > $@.tmp - mv $@.tmp $@ - -MOSTLYCLEANFILES = \ - msgpack/type/tuple.hpp \ - msgpack/type/define.hpp \ - msgpack/zone.hpp - -EXTRA_DIST = \ - msgpack/type/tuple.hpp.erb \ - msgpack/type/define.hpp.erb \ - msgpack/zone.hpp.erb - libmsgpack_la_LIBADD = -L../c -lmsgpackc # -version-info CURRENT:REVISION:AGE From dba7f480edc6302eb9938bd3388a5042ba3c499a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 19 Aug 2009 17:50:15 +0900 Subject: [PATCH 0288/1172] cpp: add missing preprocess.sh file --- cpp/preprocess.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100755 cpp/preprocess.sh diff --git a/cpp/preprocess.sh b/cpp/preprocess.sh new file mode 100755 index 0000000..69d3b96 --- /dev/null +++ b/cpp/preprocess.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +function preprocess() { + erb $1.erb > $1.tmp + mv $1.tmp $1 +} + +preprocess msgpack/type/tuple.hpp +preprocess msgpack/type/define.hpp +preprocess msgpack/zone.hpp + From 1ba330c4735082d917099fd8eb5fa1f89626cf93 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 25 Aug 2009 18:25:58 +0900 Subject: [PATCH 0289/1172] fix cpp/preprocess.sh --- cpp/preprocess.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/preprocess.sh b/cpp/preprocess.sh index 69d3b96..332dd9c 100755 --- a/cpp/preprocess.sh +++ b/cpp/preprocess.sh @@ -1,6 +1,6 @@ #!/bin/sh -function preprocess() { +preprocess() { erb $1.erb > $1.tmp mv $1.tmp $1 } From d3f9ab7deced474d88bb5b5e85bdb54b312ad86c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 25 Aug 2009 18:35:26 +0900 Subject: [PATCH 0290/1172] add c/test.cpp cpp/test.cpp --- c/Makefile.am | 8 ++ c/test.cpp | 28 +++++++ cpp/Makefile.am | 8 ++ cpp/test.cpp | 203 ++++-------------------------------------------- 4 files changed, 57 insertions(+), 190 deletions(-) create mode 100644 c/test.cpp diff --git a/c/Makefile.am b/c/Makefile.am index daa8a76..0a52446 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -18,3 +18,11 @@ nobase_include_HEADERS = \ # -version-info CURRENT:REVISION:AGE libmsgpackc_la_LDFLAGS = -version-info 1:0:0 +check_PROGRAMS = \ + msgpackc_test + +msgpackc_test_SOURCES = test.cpp +msgpackc_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c +msgpackc_test_LDFLAGS = libmsgpackc.la -lgtest_main + +TESTS = $(check_PROGRAMS) diff --git a/c/test.cpp b/c/test.cpp new file mode 100644 index 0000000..b42d931 --- /dev/null +++ b/c/test.cpp @@ -0,0 +1,28 @@ +#include "msgpack.h" + +#include + +TEST(MSGPACKC, simple_buffer) +{ + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + + msgpack_pack_int(pk, 1); + + msgpack_zone z; + msgpack_zone_init(&z, 2048); + + msgpack_object obj; + + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + + EXPECT_EQ(ret, MSGPACK_UNPACK_SUCCESS); + + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); + EXPECT_EQ(1, obj.via.u64); +} + diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 2c11535..af7e6e2 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -32,3 +32,11 @@ libmsgpack_la_LIBADD = -L../c -lmsgpackc # -version-info CURRENT:REVISION:AGE libmsgpack_la_LDFLAGS = -version-info 1:0:0 +check_PROGRAMS = \ + msgpack_test + +msgpackc_test_SOURCES = test.cpp +msgpackc_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -I$(top_srcdir)/cpp +msgpackc_test_LDFLAGS = libmsgpack.la -lgtest_main + +TESTS = $(check_PROGRAMS) diff --git a/cpp/test.cpp b/cpp/test.cpp index dd0b1fd..1c5e6c8 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -1,201 +1,24 @@ -#include -#include -#include -#include -#include +#include "msgpack.hpp" -using namespace msgpack; +#include -class checker { -public: - template - void check(const char* d, size_t len, T should) { - try { - std::cout << "----" << std::endl; - - object o; - try { - o = unpack(d, len, m_zone); - } catch (std::runtime_error& e) { - std::cout << o << std::endl; - std::cout << "**" << e.what() << "**" << std::endl; - return; - } - - std::cout << o << std::endl; - - try { - std::stringstream s; - pack(s, should); - std::string str(s.str()); - object ro = unpack(str.data(), str.size(), m_zone); - std::cout << ro << std::endl; - if(ro != o) { throw std::runtime_error("NOT MATCH"); } - } catch (std::runtime_error& e) { - std::cout << "** REUNPACK FAILED **" << std::endl; - std::cout << e.what() << std::endl; - } catch (...) { - std::cout << "** REUNPACK FAILED **" << std::endl; - std::cout << "unknown error" << std::endl; - } - - } catch (...) { m_zone.clear(); throw; } - m_zone.clear(); - } -private: - zone m_zone; -}; - -int main(void) +TEST(MSGPACKC, simple_buffer) { - checker c; + msgpack::sbuffer sbuf; -#if 0 - { // SimpleValue - const char d[] = { - 0x93, 0xc0, 0xc2, 0xc3, - }; - c.check(d, sizeof(d), - type::make_tuple( - type::nil(), false, true - ) - ); - } + int v = 0; - { // Fixnum - const char d[] = { - 0x92, - 0x93, 0x00, 0x40, 0x7f, - 0x93, 0xe0, 0xf0, 0xff, - }; - c.check(d, sizeof(d), - type::make_tuple( - type::make_tuple( - 0, 64, 127 - ), - type::make_tuple( - -32, -16, -1 - ) - ) - ); - } + msgpack::pack(sbuf, v); - { // FixArray - const char d[] = { - 0x92, - 0x90, - 0x91, - 0x91, 0xc0, - }; - std::vector empty; - c.check(d, sizeof(d), - type::make_tuple( - empty, - type::make_tuple( - type::make_tuple( - type::nil() - ) - ) - ) - ); - } + msgpack::zone z; + msgpack::object obj; - { // FixRaw - const char d[] = { - 0x94, - 0xa0, - 0xa1, 'a', - 0xa2, 'b', 'c', - 0xa3, 'd', 'e', 'f', - }; - c.check(d, sizeof(d), - type::make_tuple( - std::string(""), - std::string("a"), - std::string("bc"), - type::raw_ref("def", 3) - ) - ); - } -#endif + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); - static const unsigned TASK_ARRAY = 1000; - static const unsigned TASK_REPEAT = 10; - std::vector task; + obj.convert(&v); - // create task - { - static char traw[64]; - memset(traw, 'a', sizeof(traw)); - - task.resize(TASK_ARRAY); - for(unsigned i=0; i < TASK_ARRAY; ++i) { - task[i] = std::string(traw, sizeof(traw)); - } - } - - - std::stringstream stream; - - // send message - { - for(unsigned i=0; i < TASK_REPEAT; ++i) { - pack(stream, task); - } - std::cout << "send " << stream.str().size() << " bytes" << std::endl; - } - - ssize_t total_bytes = stream.str().size(); - stream.seekg(0); - - // reserive message - { - unsigned num_msg = 0; - static const size_t RESERVE_SIZE = 32;//*1024; - - unpacker pac; - - while(stream.good() && total_bytes > 0) { - - // 1. reserve buffer - pac.reserve_buffer(RESERVE_SIZE); - - // 2. read data to buffer() up to buffer_capacity() bytes - size_t sz = stream.readsome( - pac.buffer(), - pac.buffer_capacity()); - - total_bytes -= sz; - std::cout << "read " << sz << " bytes to capacity " - << pac.buffer_capacity() << " bytes" - << std::endl; - - // 3. specify the number of bytes actually copied - pac.buffer_consumed(sz); - - // 4. repeat execute() until it returns false - while( pac.execute() ) { - // 5.1. take out the parsed object - object o = pac.data(); - - // 5.2 release the zone - std::auto_ptr olife( pac.release_zone() ); - - // 5.3 re-initialize the unpacker */ - pac.reset(); - - // do some with the o and olife - std::cout << "message parsed: " << o << std::endl; - ++num_msg; - } - - } - - std::cout << "stream finished" << std::endl; - std::cout << num_msg << " messages reached" << std::endl; - } - - return 0; + EXPECT_EQ(0, v); } - From 51e435d46c481755a8ffca50a0cffa038bd5db08 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Tue, 25 Aug 2009 18:40:06 +0900 Subject: [PATCH 0291/1172] fix build --- c/test.cpp | 2 +- cpp/Makefile.am | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/c/test.cpp b/c/test.cpp index b42d931..bc418ed 100644 --- a/c/test.cpp +++ b/c/test.cpp @@ -10,7 +10,7 @@ TEST(MSGPACKC, simple_buffer) msgpack_packer pk; msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); - msgpack_pack_int(pk, 1); + msgpack_pack_int(&pk, 1); msgpack_zone z; msgpack_zone_init(&z, 2048); diff --git a/cpp/Makefile.am b/cpp/Makefile.am index af7e6e2..42d6d2a 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -35,8 +35,8 @@ libmsgpack_la_LDFLAGS = -version-info 1:0:0 check_PROGRAMS = \ msgpack_test -msgpackc_test_SOURCES = test.cpp -msgpackc_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -I$(top_srcdir)/cpp -msgpackc_test_LDFLAGS = libmsgpack.la -lgtest_main +msgpack_test_SOURCES = test.cpp +msgpack_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -I$(top_srcdir)/cpp +msgpack_test_LDFLAGS = libmsgpack.la -lgtest_main TESTS = $(check_PROGRAMS) From 5b8777026a288a1d7606f0a2beac3dbc62915dbc Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Tue, 25 Aug 2009 19:04:21 +0900 Subject: [PATCH 0292/1172] c: add sometests for serialization --- c/test.cpp | 129 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 113 insertions(+), 16 deletions(-) diff --git a/c/test.cpp b/c/test.cpp index bc418ed..68c4f22 100644 --- a/c/test.cpp +++ b/c/test.cpp @@ -1,28 +1,125 @@ #include "msgpack.h" +#include +#include + #include -TEST(MSGPACKC, simple_buffer) +using namespace std; + +#define LOOP 10000 + +TEST(MSGPACKC, simple_buffer_short) { - msgpack_sbuffer sbuf; - msgpack_sbuffer_init(&sbuf); + vector v; + v.push_back(0); + v.push_back(1); + v.push_back(-1); + v.push_back(numeric_limits::min()); + v.push_back(numeric_limits::max()); + for (unsigned int i = 0; i < LOOP; i++) + v.push_back(rand()); - msgpack_packer pk; - msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + for (unsigned int i = 0; i < v.size(); i++) { + short val = v[i]; + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); - msgpack_pack_int(&pk, 1); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_short(&pk, val); - msgpack_zone z; - msgpack_zone_init(&z, 2048); + msgpack_zone z; + msgpack_zone_init(&z, 2048); - msgpack_object obj; + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); - msgpack_unpack_return ret = - msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); - - EXPECT_EQ(ret, MSGPACK_UNPACK_SUCCESS); - - EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); - EXPECT_EQ(1, obj.via.u64); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + if (val < 0) { + EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, obj.type); + EXPECT_EQ(val, obj.via.i64); + } else { + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); + EXPECT_EQ(val, obj.via.u64); + } + } } +TEST(MSGPACKC, simple_buffer_int) +{ + vector v; + v.push_back(0); + v.push_back(1); + v.push_back(-1); + v.push_back(numeric_limits::min()); + v.push_back(numeric_limits::max()); + for (unsigned int i = 0; i < LOOP; i++) + v.push_back(rand()); + + for (unsigned int i = 0; i < v.size(); i++) { + int val = v[i]; + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_int(&pk, val); + + msgpack_zone z; + msgpack_zone_init(&z, 2048); + + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + if (val < 0) { + EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, obj.type); + EXPECT_EQ(val, obj.via.i64); + } else { + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); + EXPECT_EQ(val, obj.via.u64); + } + } +} + +TEST(MSGPACKC, simple_buffer_long) +{ + vector v; + v.push_back(0); + v.push_back(1); + v.push_back(-1); + v.push_back(numeric_limits::min()); + v.push_back(numeric_limits::max()); + for (unsigned int i = 0; i < LOOP; i++) + v.push_back(rand()); + + for (unsigned int i = 0; i < v.size() ; i++) { + long val = v[i]; + + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_int(&pk, val); + + msgpack_zone z; + msgpack_zone_init(&z, 2048); + + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + if (val < 0) { + EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, obj.type); + EXPECT_EQ(val, obj.via.i64); + } else { + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); + EXPECT_EQ(val, obj.via.u64); + } + } +} From c232e91f831c1934c8c805d51cfd41f94852778a Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Tue, 25 Aug 2009 19:26:54 +0900 Subject: [PATCH 0293/1172] c: fix bugs in c/test.cpp and add more tests --- c/test.cpp | 235 +++++++++++++++++++++++++++++------------------------ 1 file changed, 131 insertions(+), 104 deletions(-) diff --git a/c/test.cpp b/c/test.cpp index 68c4f22..a0fc7e4 100644 --- a/c/test.cpp +++ b/c/test.cpp @@ -7,119 +7,146 @@ using namespace std; -#define LOOP 10000 +const unsigned int kLoop = 10000; + +#define GEN_TEST_SIGNED(test_type, func_type) \ + do { \ + vector v; \ + v.push_back(0); \ + v.push_back(1); \ + v.push_back(-1); \ + v.push_back(numeric_limits::min()); \ + v.push_back(numeric_limits::max()); \ + for (unsigned int i = 0; i < kLoop; i++) \ + v.push_back(rand()); \ + for (unsigned int i = 0; i < v.size() ; i++) { \ + test_type val = v[i]; \ + msgpack_sbuffer sbuf; \ + msgpack_sbuffer_init(&sbuf); \ + msgpack_packer pk; \ + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); \ + msgpack_pack_##func_type(&pk, val); \ + msgpack_zone z; \ + msgpack_zone_init(&z, 2048); \ + msgpack_object obj; \ + msgpack_unpack_return ret = \ + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); \ + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); \ + if (val < 0) { \ + EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, obj.type); \ + EXPECT_EQ(val, obj.via.i64); \ + } else { \ + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); \ + EXPECT_EQ(val, obj.via.u64); \ + } \ + EXPECT_EQ(val, obj.via.u64); \ + } \ + } while(0) + +#define GEN_TEST_UNSIGNED(test_type, func_type) \ + do { \ + vector v; \ + v.push_back(0); \ + v.push_back(1); \ + v.push_back(-1); \ + v.push_back(numeric_limits::min()); \ + v.push_back(numeric_limits::max()); \ + for (unsigned int i = 0; i < kLoop; i++) \ + v.push_back(rand()); \ + for (unsigned int i = 0; i < v.size() ; i++) { \ + test_type val = v[i]; \ + msgpack_sbuffer sbuf; \ + msgpack_sbuffer_init(&sbuf); \ + msgpack_packer pk; \ + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); \ + msgpack_pack_##func_type(&pk, val); \ + msgpack_zone z; \ + msgpack_zone_init(&z, 2048); \ + msgpack_object obj; \ + msgpack_unpack_return ret = \ + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); \ + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); \ + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); \ + EXPECT_EQ(val, obj.via.u64); \ + } \ + } while(0) TEST(MSGPACKC, simple_buffer_short) { - vector v; - v.push_back(0); - v.push_back(1); - v.push_back(-1); - v.push_back(numeric_limits::min()); - v.push_back(numeric_limits::max()); - for (unsigned int i = 0; i < LOOP; i++) - v.push_back(rand()); - - for (unsigned int i = 0; i < v.size(); i++) { - short val = v[i]; - msgpack_sbuffer sbuf; - msgpack_sbuffer_init(&sbuf); - - msgpack_packer pk; - msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); - msgpack_pack_short(&pk, val); - - msgpack_zone z; - msgpack_zone_init(&z, 2048); - - msgpack_object obj; - msgpack_unpack_return ret = - msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); - - EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); - if (val < 0) { - EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, obj.type); - EXPECT_EQ(val, obj.via.i64); - } else { - EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); - EXPECT_EQ(val, obj.via.u64); - } - } + GEN_TEST_SIGNED(short, short); } TEST(MSGPACKC, simple_buffer_int) { - vector v; - v.push_back(0); - v.push_back(1); - v.push_back(-1); - v.push_back(numeric_limits::min()); - v.push_back(numeric_limits::max()); - for (unsigned int i = 0; i < LOOP; i++) - v.push_back(rand()); - - for (unsigned int i = 0; i < v.size(); i++) { - int val = v[i]; - msgpack_sbuffer sbuf; - msgpack_sbuffer_init(&sbuf); - - msgpack_packer pk; - msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); - msgpack_pack_int(&pk, val); - - msgpack_zone z; - msgpack_zone_init(&z, 2048); - - msgpack_object obj; - msgpack_unpack_return ret = - msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); - - EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); - if (val < 0) { - EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, obj.type); - EXPECT_EQ(val, obj.via.i64); - } else { - EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); - EXPECT_EQ(val, obj.via.u64); - } - } + GEN_TEST_SIGNED(int, int); } TEST(MSGPACKC, simple_buffer_long) { - vector v; - v.push_back(0); - v.push_back(1); - v.push_back(-1); - v.push_back(numeric_limits::min()); - v.push_back(numeric_limits::max()); - for (unsigned int i = 0; i < LOOP; i++) - v.push_back(rand()); - - for (unsigned int i = 0; i < v.size() ; i++) { - long val = v[i]; - - msgpack_sbuffer sbuf; - msgpack_sbuffer_init(&sbuf); - - msgpack_packer pk; - msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); - msgpack_pack_int(&pk, val); - - msgpack_zone z; - msgpack_zone_init(&z, 2048); - - msgpack_object obj; - msgpack_unpack_return ret = - msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); - - EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); - if (val < 0) { - EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, obj.type); - EXPECT_EQ(val, obj.via.i64); - } else { - EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); - EXPECT_EQ(val, obj.via.u64); - } - } + GEN_TEST_SIGNED(long, long); +} + +TEST(MSGPACKC, simple_buffer_long_long) +{ + GEN_TEST_SIGNED(long long, long_long); +} + +TEST(MSGPACKC, simple_buffer_unsigned_short) +{ + GEN_TEST_UNSIGNED(unsigned short, unsigned_short); +} + +TEST(MSGPACKC, simple_buffer_unsigned_int) +{ + GEN_TEST_UNSIGNED(unsigned int, unsigned_int); +} + +TEST(MSGPACKC, simple_buffer_unsigned_long) +{ + GEN_TEST_UNSIGNED(unsigned long, unsigned_long); +} + +TEST(MSGPACKC, simple_buffer_unsigned_long_long) +{ + GEN_TEST_UNSIGNED(unsigned long long, unsigned_long_long); +} + +TEST(MSGPACKC, simple_buffer_uint8) +{ + GEN_TEST_UNSIGNED(uint8_t, uint8); +} + +TEST(MSGPACKC, simple_buffer_uint16) +{ + GEN_TEST_UNSIGNED(uint16_t, uint16); +} + +TEST(MSGPACKC, simple_buffer_uint32) +{ + GEN_TEST_UNSIGNED(uint32_t, uint32); +} + +TEST(MSGPACKC, simple_buffer_uint64) +{ + GEN_TEST_UNSIGNED(uint64_t, uint64); +} + +TEST(MSGPACKC, simple_buffer_int8) +{ + GEN_TEST_SIGNED(int8_t, int8); +} + +TEST(MSGPACKC, simple_buffer_int16) +{ + GEN_TEST_SIGNED(int16_t, int16); +} + +TEST(MSGPACKC, simple_buffer_int32) +{ + GEN_TEST_SIGNED(int32_t, int32); +} + +TEST(MSGPACKC, simple_buffer_int64) +{ + GEN_TEST_SIGNED(int64_t, int64); } From 93745710569a9c0051ddce62da7002dbde3c742b Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Wed, 26 Aug 2009 09:24:07 +0900 Subject: [PATCH 0294/1172] C: add more tests (float, double, nil, true, false, array, raw) --- c/test.cpp | 203 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 201 insertions(+), 2 deletions(-) diff --git a/c/test.cpp b/c/test.cpp index a0fc7e4..faeb698 100644 --- a/c/test.cpp +++ b/c/test.cpp @@ -1,5 +1,6 @@ #include "msgpack.h" +#include #include #include @@ -8,6 +9,7 @@ using namespace std; const unsigned int kLoop = 10000; +const double kEPS = 10e-10; #define GEN_TEST_SIGNED(test_type, func_type) \ do { \ @@ -39,7 +41,6 @@ const unsigned int kLoop = 10000; EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); \ EXPECT_EQ(val, obj.via.u64); \ } \ - EXPECT_EQ(val, obj.via.u64); \ } \ } while(0) @@ -48,7 +49,7 @@ const unsigned int kLoop = 10000; vector v; \ v.push_back(0); \ v.push_back(1); \ - v.push_back(-1); \ + v.push_back(2); \ v.push_back(numeric_limits::min()); \ v.push_back(numeric_limits::max()); \ for (unsigned int i = 0; i < kLoop; i++) \ @@ -150,3 +151,201 @@ TEST(MSGPACKC, simple_buffer_int64) { GEN_TEST_SIGNED(int64_t, int64); } + +TEST(MSGPACKC, simple_buffer_float) +{ + vector v; + v.push_back(0.0); + v.push_back(1); + v.push_back(-1); + v.push_back(numeric_limits::min()); + v.push_back(numeric_limits::max()); + v.push_back(nanf("tag")); + v.push_back(1.0/0.0); // inf + for (unsigned int i = 0; i < kLoop; i++) + v.push_back(drand48()); + + for (unsigned int i = 0; i < v.size() ; i++) { + float val = v[i]; + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_float(&pk, val); + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_DOUBLE, obj.type); + if (isnan(val)) + EXPECT_TRUE(isnan(obj.via.dec)); + else if (isinf(val)) + EXPECT_TRUE(isinf(obj.via.dec)); + else + EXPECT_TRUE(fabs(obj.via.dec - val) <= kEPS); + } +} + +TEST(MSGPACKC, simple_buffer_double) +{ + vector v; + v.push_back(0.0); + v.push_back(1); + v.push_back(-1); + v.push_back(numeric_limits::min()); + v.push_back(numeric_limits::max()); + v.push_back(nan("tag")); + v.push_back(1.0/0.0); // inf + for (unsigned int i = 0; i < kLoop; i++) + v.push_back(drand48()); + + for (unsigned int i = 0; i < v.size() ; i++) { + double val = v[i]; + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_double(&pk, val); + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_DOUBLE, obj.type); + if (isnan(val)) + EXPECT_TRUE(isnan(obj.via.dec)); + else if (isinf(val)) + EXPECT_TRUE(isinf(obj.via.dec)); + else + EXPECT_TRUE(fabs(obj.via.dec - val) <= kEPS); + } +} + +TEST(MSGPACKC, simple_buffer_nil) +{ + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_nil(&pk); + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_NIL, obj.type); +} + +TEST(MSGPACKC, simple_buffer_true) +{ + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_true(&pk); + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, obj.type); + EXPECT_EQ(true, obj.via.boolean); +} + +TEST(MSGPACKC, simple_buffer_false) +{ + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_false(&pk); + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, obj.type); + EXPECT_EQ(false, obj.via.boolean); +} + +TEST(MSGPACKC, simple_buffer_array) +{ + unsigned int array_size = 5; + + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_array(&pk, array_size); + msgpack_pack_nil(&pk); + msgpack_pack_true(&pk); + msgpack_pack_false(&pk); + msgpack_pack_int(&pk, 10); + msgpack_pack_int(&pk, -10); + + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret; + ret = msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_ARRAY, obj.type); + EXPECT_EQ(array_size, obj.via.array.size); + + for (unsigned int i = 0; i < obj.via.array.size; i++) { + msgpack_object o = obj.via.array.ptr[i]; + switch (i) { + case 0: + EXPECT_EQ(MSGPACK_OBJECT_NIL, o.type); + break; + case 1: + EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, o.type); + EXPECT_EQ(true, o.via.boolean); + break; + case 2: + EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, o.type); + EXPECT_EQ(false, o.via.boolean); + break; + case 3: + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, o.type); + EXPECT_EQ(10, o.via.u64); + break; + case 4: + EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, o.type); + EXPECT_EQ(-10, o.via.u64); + break; + } + } +} + +TEST(MSGPACKC, simple_buffer_raw) +{ + unsigned int raw_size = 7; + + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_raw(&pk, raw_size); + msgpack_pack_raw_body(&pk, "fr", 2); + msgpack_pack_raw_body(&pk, "syuki", 5); + // invalid data + msgpack_pack_raw_body(&pk, "", 0); + msgpack_pack_raw_body(&pk, "kzk", 0); + + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret; + ret = msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_RAW, obj.type); + EXPECT_EQ(raw_size, obj.via.raw.size); + EXPECT_EQ(0, memcmp("frsyuki", obj.via.raw.ptr, raw_size)); +} From c94772104d70290f438cf629ff693a8d7e69f827 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Wed, 26 Aug 2009 11:28:54 +0900 Subject: [PATCH 0295/1172] C: add test for map C++: add tests for primitive types, stl types, user-defined type --- c/test.cpp | 47 +++++- cpp/test.cpp | 455 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 482 insertions(+), 20 deletions(-) diff --git a/c/test.cpp b/c/test.cpp index faeb698..28a3d9e 100644 --- a/c/test.cpp +++ b/c/test.cpp @@ -9,7 +9,7 @@ using namespace std; const unsigned int kLoop = 10000; -const double kEPS = 10e-10; +const double kEPS = 1e-10; #define GEN_TEST_SIGNED(test_type, func_type) \ do { \ @@ -277,7 +277,7 @@ TEST(MSGPACKC, simple_buffer_false) TEST(MSGPACKC, simple_buffer_array) { unsigned int array_size = 5; - + msgpack_sbuffer sbuf; msgpack_sbuffer_init(&sbuf); msgpack_packer pk; @@ -324,6 +324,49 @@ TEST(MSGPACKC, simple_buffer_array) } } +TEST(MSGPACKC, simple_buffer_map) +{ + unsigned int map_size = 2; + + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_map(&pk, map_size); + msgpack_pack_true(&pk); + msgpack_pack_false(&pk); + msgpack_pack_int(&pk, 10); + msgpack_pack_int(&pk, -10); + + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret; + ret = msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_MAP, obj.type); + EXPECT_EQ(map_size, obj.via.map.size); + + for (unsigned int i = 0; i < map_size; i++) { + msgpack_object key = obj.via.map.ptr[i].key; + msgpack_object val = obj.via.map.ptr[i].val; + switch (i) { + case 0: + EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, key.type); + EXPECT_EQ(true, key.via.boolean); + EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, val.type); + EXPECT_EQ(false, val.via.boolean); + break; + case 1: + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, key.type); + EXPECT_EQ(10, key.via.u64); + EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, val.type); + EXPECT_EQ(-10, val.via.i64); + break; + } + } +} + TEST(MSGPACKC, simple_buffer_raw) { unsigned int raw_size = 7; diff --git a/cpp/test.cpp b/cpp/test.cpp index 1c5e6c8..f2ff1ab 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -1,24 +1,443 @@ #include "msgpack.hpp" +#include +#include +#include +#include +#include +#include +#include + #include -TEST(MSGPACKC, simple_buffer) +using namespace std; + +const unsigned int kLoop = 10000; +const unsigned int kElements = 100; +const double kEPS = 1e-10; + +#define GEN_TEST(test_type) \ + do { \ + vector v; \ + v.push_back(0); \ + v.push_back(1); \ + v.push_back(2); \ + v.push_back(numeric_limits::min()); \ + v.push_back(numeric_limits::max()); \ + for (unsigned int i = 0; i < kLoop; i++) \ + v.push_back(rand()); \ + for (unsigned int i = 0; i < v.size() ; i++) { \ + msgpack::sbuffer sbuf; \ + test_type val1 = v[i]; \ + msgpack::pack(sbuf, val1); \ + msgpack::zone z; \ + msgpack::object obj; \ + msgpack::unpack_return ret = \ + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); \ + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); \ + test_type val2; \ + obj.convert(&val2); \ + EXPECT_EQ(val1, val2); \ + } \ +} while(0) + +TEST(MSGPACK, simple_buffer_short) { - msgpack::sbuffer sbuf; - - int v = 0; - - msgpack::pack(sbuf, v); - - msgpack::zone z; - msgpack::object obj; - - msgpack::unpack_return ret = - msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); - - obj.convert(&v); - - EXPECT_EQ(0, v); + GEN_TEST(short); +} + +TEST(MSGPACK, simple_buffer_int) +{ + GEN_TEST(int); +} + +TEST(MSGPACK, simple_buffer_long) +{ + GEN_TEST(long); +} + +TEST(MSGPACK, simple_buffer_long_long) +{ + GEN_TEST(long long); +} + +TEST(MSGPACK, simple_buffer_unsigned_short) +{ + GEN_TEST(unsigned short); +} + +TEST(MSGPACK, simple_buffer_unsigned_int) +{ + GEN_TEST(unsigned int); +} + +TEST(MSGPACK, simple_buffer_unsigned_long) +{ + GEN_TEST(unsigned long); +} + +TEST(MSGPACK, simple_buffer_unsigned_long_long) +{ + GEN_TEST(unsigned long long); +} + +TEST(MSGPACK, simple_buffer_uint8) +{ + GEN_TEST(uint8_t); +} + +TEST(MSGPACK, simple_buffer_uint16) +{ + GEN_TEST(uint16_t); +} + +TEST(MSGPACK, simple_buffer_uint32) +{ + GEN_TEST(uint32_t); +} + +TEST(MSGPACK, simple_buffer_uint64) +{ + GEN_TEST(uint64_t); +} + +TEST(MSGPACK, simple_buffer_int8) +{ + GEN_TEST(int8_t); +} + +TEST(MSGPACK, simple_buffer_int16) +{ + GEN_TEST(int16_t); +} + +TEST(MSGPACK, simple_buffer_int32) +{ + GEN_TEST(int32_t); +} + +TEST(MSGPACK, simple_buffer_int64) +{ + GEN_TEST(int64_t); +} + +TEST(MSGPACK, simple_buffer_float) +{ + vector v; + v.push_back(0); + v.push_back(1); + v.push_back(2); + v.push_back(numeric_limits::min()); + v.push_back(numeric_limits::max()); + v.push_back(nanf("tag")); + v.push_back(1.0/0.0); // inf + for (unsigned int i = 0; i < kLoop; i++) + v.push_back(drand48()); + for (unsigned int i = 0; i < v.size() ; i++) { + msgpack::sbuffer sbuf; + float val1 = v[i]; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + float val2; + obj.convert(&val2); + + if (isnan(val1)) + EXPECT_TRUE(isnan(val2)); + else if (isinf(val1)) + EXPECT_TRUE(isinf(val2)); + else + EXPECT_TRUE(fabs(val2 - val1) <= kEPS); + } +} + +TEST(MSGPACK, simple_buffer_double) +{ + vector v; + v.push_back(0); + v.push_back(1); + v.push_back(2); + v.push_back(numeric_limits::min()); + v.push_back(numeric_limits::max()); + v.push_back(nanf("tag")); + v.push_back(1.0/0.0); // inf + for (unsigned int i = 0; i < kLoop; i++) + v.push_back(drand48()); + for (unsigned int i = 0; i < v.size() ; i++) { + msgpack::sbuffer sbuf; + double val1 = v[i]; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + double val2; + obj.convert(&val2); + + if (isnan(val1)) + EXPECT_TRUE(isnan(val2)); + else if (isinf(val1)) + EXPECT_TRUE(isinf(val2)); + else + EXPECT_TRUE(fabs(val2 - val1) <= kEPS); + } +} + +TEST(MSGPACK, simple_buffer_true) +{ + msgpack::sbuffer sbuf; + bool val1 = true; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + bool val2; + obj.convert(&val2); + EXPECT_EQ(val1, val2); +} + +TEST(MSGPACK, simple_buffer_false) +{ + msgpack::sbuffer sbuf; + bool val1 = false; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + bool val2; + obj.convert(&val2); + EXPECT_EQ(val1, val2); +} + +//----------------------------------------------------------------------------- + +TEST(MSGPACK_STL, simple_buffer_string) +{ + for (unsigned int k = 0; k < kLoop; k++) { + string val1; + for (unsigned int i = 0; i < kElements; i++) + val1 += 'a' + rand() % 26; + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + string val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_EQ(val1, val2); + } +} + +TEST(MSGPACK_STL, simple_buffer_vector) +{ + for (unsigned int k = 0; k < kLoop; k++) { + vector val1; + for (unsigned int i = 0; i < kElements; i++) + val1.push_back(rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + vector val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_TRUE(equal(val1.begin(), val1.end(), val2.begin())); + } +} + +TEST(MSGPACK_STL, simple_buffer_map) +{ + for (unsigned int k = 0; k < kLoop; k++) { + map val1; + for (unsigned int i = 0; i < kElements; i++) + val1[rand()] = rand(); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + map val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_TRUE(equal(val1.begin(), val1.end(), val2.begin())); + } +} + +TEST(MSGPACK_STL, simple_buffer_deque) +{ + for (unsigned int k = 0; k < kLoop; k++) { + deque val1; + for (unsigned int i = 0; i < kElements; i++) + val1.push_back(rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + deque val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_TRUE(equal(val1.begin(), val1.end(), val2.begin())); + } +} + +TEST(MSGPACK_STL, simple_buffer_list) +{ + for (unsigned int k = 0; k < kLoop; k++) { + list val1; + for (unsigned int i = 0; i < kElements; i++) + val1.push_back(rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + list val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_TRUE(equal(val1.begin(), val1.end(), val2.begin())); + } +} + +TEST(MSGPACK_STL, simple_buffer_set) +{ + for (unsigned int k = 0; k < kLoop; k++) { + set val1; + for (unsigned int i = 0; i < kElements; i++) + val1.insert(rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + set val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_TRUE(equal(val1.begin(), val1.end(), val2.begin())); + } +} + +TEST(MSGPACK_STL, simple_buffer_pair) +{ + for (unsigned int k = 0; k < kLoop; k++) { + pair val1 = make_pair(rand(), rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + pair val2; + obj.convert(&val2); + EXPECT_EQ(val1.first, val2.first); + EXPECT_EQ(val1.second, val2.second); + } +} + +class TestClass +{ +public: + TestClass() : i(0), s("kzk") {} + int i; + string s; + MSGPACK_DEFINE(i, s); +}; + +TEST(MSGPACK_USER_DEFINED, simple_buffer_class) +{ + for (unsigned int k = 0; k < kLoop; k++) { + TestClass val1; + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + TestClass val2; + val2.i = -1; + val2.s = ""; + obj.convert(&val2); + EXPECT_EQ(val1.i, val2.i); + EXPECT_EQ(val1.s, val2.s); + } +} + +class TestClass2 +{ +public: + TestClass2() : i(0), s("kzk") { + for (unsigned int i = 0; i < kElements; i++) + v.push_back(rand()); + } + int i; + string s; + vector v; + MSGPACK_DEFINE(i, s, v); +}; + +TEST(MSGPACK_USER_DEFINED, simple_buffer_class_old_to_new) +{ + for (unsigned int k = 0; k < kLoop; k++) { + TestClass val1; + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + TestClass2 val2; + val2.i = -1; + val2.s = ""; + val2.v = vector(); + obj.convert(&val2); + EXPECT_EQ(val1.i, val2.i); + EXPECT_EQ(val1.s, val2.s); + EXPECT_FALSE(val2.s.empty()); + } +} + +TEST(MSGPACK_USER_DEFINED, simple_buffer_class_new_to_old) +{ + for (unsigned int k = 0; k < kLoop; k++) { + TestClass2 val1; + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + TestClass val2; + val2.i = -1; + val2.s = ""; + obj.convert(&val2); + EXPECT_EQ(val1.i, val2.i); + EXPECT_EQ(val1.s, val2.s); + EXPECT_FALSE(val2.s.empty()); + } } From 7186edc45e057aff9f10511547c3f94c16c9c773 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Wed, 26 Aug 2009 11:52:12 +0900 Subject: [PATCH 0296/1172] C: more strict tests for float, double C++: more strict tests for float, double & enum, union member --- c/test.cpp | 19 ++++--- cpp/test.cpp | 141 +++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 145 insertions(+), 15 deletions(-) diff --git a/c/test.cpp b/c/test.cpp index 28a3d9e..0277206 100644 --- a/c/test.cpp +++ b/c/test.cpp @@ -156,14 +156,17 @@ TEST(MSGPACKC, simple_buffer_float) { vector v; v.push_back(0.0); - v.push_back(1); - v.push_back(-1); + v.push_back(1.0); + v.push_back(-1.0); v.push_back(numeric_limits::min()); v.push_back(numeric_limits::max()); v.push_back(nanf("tag")); v.push_back(1.0/0.0); // inf - for (unsigned int i = 0; i < kLoop; i++) + v.push_back(-(1.0/0.0)); // -inf + for (unsigned int i = 0; i < kLoop; i++) { v.push_back(drand48()); + v.push_back(-drand48()); + } for (unsigned int i = 0; i < v.size() ; i++) { float val = v[i]; @@ -192,14 +195,18 @@ TEST(MSGPACKC, simple_buffer_double) { vector v; v.push_back(0.0); - v.push_back(1); - v.push_back(-1); + v.push_back(-0.0); + v.push_back(1.0); + v.push_back(-1.0); v.push_back(numeric_limits::min()); v.push_back(numeric_limits::max()); v.push_back(nan("tag")); v.push_back(1.0/0.0); // inf - for (unsigned int i = 0; i < kLoop; i++) + v.push_back(-(1.0/0.0)); // -inf + for (unsigned int i = 0; i < kLoop; i++) { v.push_back(drand48()); + v.push_back(-drand48()); + } for (unsigned int i = 0; i < v.size() ; i++) { double val = v[i]; diff --git a/cpp/test.cpp b/cpp/test.cpp index f2ff1ab..fc3f9b4 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -124,15 +124,19 @@ TEST(MSGPACK, simple_buffer_int64) TEST(MSGPACK, simple_buffer_float) { vector v; - v.push_back(0); - v.push_back(1); - v.push_back(2); + v.push_back(0.0); + v.push_back(-0.0); + v.push_back(1.0); + v.push_back(-1.0); v.push_back(numeric_limits::min()); v.push_back(numeric_limits::max()); v.push_back(nanf("tag")); v.push_back(1.0/0.0); // inf - for (unsigned int i = 0; i < kLoop; i++) + v.push_back(-(1.0/0.0)); // -inf + for (unsigned int i = 0; i < kLoop; i++) { v.push_back(drand48()); + v.push_back(-drand48()); + } for (unsigned int i = 0; i < v.size() ; i++) { msgpack::sbuffer sbuf; float val1 = v[i]; @@ -157,15 +161,19 @@ TEST(MSGPACK, simple_buffer_float) TEST(MSGPACK, simple_buffer_double) { vector v; - v.push_back(0); - v.push_back(1); - v.push_back(2); + v.push_back(0.0); + v.push_back(-0.0); + v.push_back(1.0); + v.push_back(-1.0); v.push_back(numeric_limits::min()); v.push_back(numeric_limits::max()); v.push_back(nanf("tag")); v.push_back(1.0/0.0); // inf - for (unsigned int i = 0; i < kLoop; i++) + v.push_back(-(1.0/0.0)); // -inf + for (unsigned int i = 0; i < kLoop; i++) { v.push_back(drand48()); + v.push_back(-drand48()); + } for (unsigned int i = 0; i < v.size() ; i++) { msgpack::sbuffer sbuf; double val1 = v[i]; @@ -383,7 +391,7 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_class) obj.convert(&val2); EXPECT_EQ(val1.i, val2.i); EXPECT_EQ(val1.s, val2.s); - } + } } class TestClass2 @@ -441,3 +449,118 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_class_new_to_old) EXPECT_FALSE(val2.s.empty()); } } + +class TestEnumMemberClass +{ +public: + TestEnumMemberClass() + : t1(STATE_A), t2(STATE_B), t3(STATE_C) {} + + enum TestEnumType { + STATE_INVALID = 0, + STATE_A = 1, + STATE_B = 2, + STATE_C = 3 + }; + TestEnumType t1; + TestEnumType t2; + TestEnumType t3; + + MSGPACK_DEFINE((int&)t1, (int&)t2, (int&)t3); +}; + +TEST(MSGPACK_USER_DEFINED, simple_buffer_enum_member) +{ + TestEnumMemberClass val1; + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + TestEnumMemberClass val2; + val2.t1 = TestEnumMemberClass::STATE_INVALID; + val2.t2 = TestEnumMemberClass::STATE_INVALID; + val2.t3 = TestEnumMemberClass::STATE_INVALID; + obj.convert(&val2); + EXPECT_EQ(val1.t1, val2.t1); + EXPECT_EQ(val1.t2, val2.t2); + EXPECT_EQ(val1.t3, val2.t3); +} + +class TestUnionMemberClass +{ +public: + TestUnionMemberClass() {} + TestUnionMemberClass(double f) { + is_double = true; + value.f = f; + } + TestUnionMemberClass(int i) { + is_double = false; + value.i = i; + } + + union { + double f; + int i; + } value; + bool is_double; + + template + void msgpack_pack(Packer& pk) const + { + if (is_double) + pk.pack(msgpack::type::tuple(true, value.f)); + else + pk.pack(msgpack::type::tuple(false, value.i)); + } + + void msgpack_unpack(msgpack::object o) + { + msgpack::type::tuple tuple; + o.convert(&tuple); + + is_double = tuple.get<0>(); + if (is_double) + tuple.get<1>().convert(&value.f); + else + tuple.get<1>().convert(&value.i); + } +}; + +TEST(MSGPACK_USER_DEFINED, simple_buffer_union_member) +{ + { + // double + TestUnionMemberClass val1(1.0); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + TestUnionMemberClass val2; + obj.convert(&val2); + EXPECT_EQ(val1.is_double, val2.is_double); + EXPECT_TRUE(fabs(val1.value.f - val2.value.f) < kEPS); + } + { + // int + TestUnionMemberClass val1(1); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + TestUnionMemberClass val2; + obj.convert(&val2); + EXPECT_EQ(val1.is_double, val2.is_double); + EXPECT_EQ(val1.value.i, 1); + EXPECT_EQ(val1.value.i, val2.value.i); + } +} From c12d5b8461fdd3f3238b37607bf8b9b08af98a1c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 Aug 2009 12:27:28 +0900 Subject: [PATCH 0297/1172] c: test.cpp: add msgpack_sbuffer_destroy and msgpack_zone_destroy --- c/test.cpp | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/c/test.cpp b/c/test.cpp index 0277206..f5646ea 100644 --- a/c/test.cpp +++ b/c/test.cpp @@ -41,6 +41,8 @@ const double kEPS = 1e-10; EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); \ EXPECT_EQ(val, obj.via.u64); \ } \ + msgpack_zone_destroy(&z); \ + msgpack_sbuffer_destroy(&sbuf); \ } \ } while(0) @@ -69,6 +71,8 @@ const double kEPS = 1e-10; EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); \ EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); \ EXPECT_EQ(val, obj.via.u64); \ + msgpack_zone_destroy(&z); \ + msgpack_sbuffer_destroy(&sbuf); \ } \ } while(0) @@ -188,6 +192,8 @@ TEST(MSGPACKC, simple_buffer_float) EXPECT_TRUE(isinf(obj.via.dec)); else EXPECT_TRUE(fabs(obj.via.dec - val) <= kEPS); + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } } @@ -228,6 +234,8 @@ TEST(MSGPACKC, simple_buffer_double) EXPECT_TRUE(isinf(obj.via.dec)); else EXPECT_TRUE(fabs(obj.via.dec - val) <= kEPS); + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } } @@ -245,6 +253,8 @@ TEST(MSGPACKC, simple_buffer_nil) msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); EXPECT_EQ(MSGPACK_OBJECT_NIL, obj.type); + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } TEST(MSGPACKC, simple_buffer_true) @@ -262,6 +272,8 @@ TEST(MSGPACKC, simple_buffer_true) EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, obj.type); EXPECT_EQ(true, obj.via.boolean); + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } TEST(MSGPACKC, simple_buffer_false) @@ -279,6 +291,8 @@ TEST(MSGPACKC, simple_buffer_false) EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, obj.type); EXPECT_EQ(false, obj.via.boolean); + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } TEST(MSGPACKC, simple_buffer_array) @@ -325,10 +339,13 @@ TEST(MSGPACKC, simple_buffer_array) break; case 4: EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, o.type); - EXPECT_EQ(-10, o.via.u64); + EXPECT_EQ(-10, o.via.i64); break; } } + + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } TEST(MSGPACKC, simple_buffer_map) @@ -369,9 +386,12 @@ TEST(MSGPACKC, simple_buffer_map) EXPECT_EQ(10, key.via.u64); EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, val.type); EXPECT_EQ(-10, val.via.i64); - break; + break; } } + + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } TEST(MSGPACKC, simple_buffer_raw) @@ -398,4 +418,7 @@ TEST(MSGPACKC, simple_buffer_raw) EXPECT_EQ(MSGPACK_OBJECT_RAW, obj.type); EXPECT_EQ(raw_size, obj.via.raw.size); EXPECT_EQ(0, memcmp("frsyuki", obj.via.raw.ptr, raw_size)); + + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } From ebf64d9892b383aa8ddef1e09ef8c904dc8f1e58 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 Aug 2009 12:28:08 +0900 Subject: [PATCH 0298/1172] cpp: test.cpp: fixes EXPECT_EQ rule --- cpp/test.cpp | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/cpp/test.cpp b/cpp/test.cpp index fc3f9b4..d9f51da 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -34,7 +34,7 @@ const double kEPS = 1e-10; msgpack::object obj; \ msgpack::unpack_return ret = \ msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); \ - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); \ + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); \ test_type val2; \ obj.convert(&val2); \ EXPECT_EQ(val1, val2); \ @@ -145,7 +145,7 @@ TEST(MSGPACK, simple_buffer_float) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); float val2; obj.convert(&val2); @@ -182,7 +182,7 @@ TEST(MSGPACK, simple_buffer_double) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); double val2; obj.convert(&val2); @@ -204,7 +204,7 @@ TEST(MSGPACK, simple_buffer_true) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); bool val2; obj.convert(&val2); EXPECT_EQ(val1, val2); @@ -219,7 +219,7 @@ TEST(MSGPACK, simple_buffer_false) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); bool val2; obj.convert(&val2); EXPECT_EQ(val1, val2); @@ -239,7 +239,7 @@ TEST(MSGPACK_STL, simple_buffer_string) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); string val2; obj.convert(&val2); EXPECT_EQ(val1.size(), val2.size()); @@ -259,7 +259,7 @@ TEST(MSGPACK_STL, simple_buffer_vector) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); vector val2; obj.convert(&val2); EXPECT_EQ(val1.size(), val2.size()); @@ -279,7 +279,7 @@ TEST(MSGPACK_STL, simple_buffer_map) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); map val2; obj.convert(&val2); EXPECT_EQ(val1.size(), val2.size()); @@ -299,7 +299,7 @@ TEST(MSGPACK_STL, simple_buffer_deque) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); deque val2; obj.convert(&val2); EXPECT_EQ(val1.size(), val2.size()); @@ -319,7 +319,7 @@ TEST(MSGPACK_STL, simple_buffer_list) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); list val2; obj.convert(&val2); EXPECT_EQ(val1.size(), val2.size()); @@ -339,7 +339,7 @@ TEST(MSGPACK_STL, simple_buffer_set) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); set val2; obj.convert(&val2); EXPECT_EQ(val1.size(), val2.size()); @@ -357,7 +357,7 @@ TEST(MSGPACK_STL, simple_buffer_pair) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); pair val2; obj.convert(&val2); EXPECT_EQ(val1.first, val2.first); @@ -384,7 +384,7 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_class) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); TestClass val2; val2.i = -1; val2.s = ""; @@ -417,7 +417,7 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_class_old_to_new) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); TestClass2 val2; val2.i = -1; val2.s = ""; @@ -439,7 +439,7 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_class_new_to_old) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); TestClass val2; val2.i = -1; val2.s = ""; @@ -478,7 +478,7 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_enum_member) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); TestEnumMemberClass val2; val2.t1 = TestEnumMemberClass::STATE_INVALID; val2.t2 = TestEnumMemberClass::STATE_INVALID; @@ -541,7 +541,7 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_union_member) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); TestUnionMemberClass val2; obj.convert(&val2); EXPECT_EQ(val1.is_double, val2.is_double); @@ -556,7 +556,7 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_union_member) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); TestUnionMemberClass val2; obj.convert(&val2); EXPECT_EQ(val1.is_double, val2.is_double); From 320c79db49fbb19b1a5097b0d254ece70cbf2c54 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Fri, 4 Sep 2009 16:20:29 +0900 Subject: [PATCH 0299/1172] c++: add vrefbuffer test --- cpp/test.cpp | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/cpp/test.cpp b/cpp/test.cpp index d9f51da..3907c47 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -564,3 +564,109 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_union_member) EXPECT_EQ(val1.value.i, val2.value.i); } } + +//----------------------------------------------------------------------------- + +#define GEN_TEST_VREF(test_type) \ + do { \ + vector v; \ + v.push_back(0); \ + for (unsigned int i = 0; i < v.size(); i++) { \ + test_type val1 = v[i]; \ + msgpack::vrefbuffer vbuf; \ + msgpack::pack(vbuf, val1); \ + msgpack::sbuffer sbuf; \ + const struct iovec* cur = vbuf.vector(); \ + const struct iovec* end = cur + vbuf.vector_size(); \ + for(; cur != end; ++cur) \ + sbuf.write((const char*)cur->iov_base, cur->iov_len); \ + msgpack::zone z; \ + msgpack::object obj; \ + msgpack::unpack_return ret = \ + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); \ + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); \ + test_type val2; \ + obj.convert(&val2); \ + EXPECT_EQ(val1, val2); \ + } \ + } while(0); + +TEST(MSGPACK, vrefbuffer_short) +{ + GEN_TEST_VREF(short); +} + +TEST(MSGPACK, vrefbuffer_int) +{ + GEN_TEST_VREF(int); +} + +TEST(MSGPACK, vrefbuffer_long) +{ + GEN_TEST_VREF(long); +} + +TEST(MSGPACK, vrefbuffer_long_long) +{ + GEN_TEST_VREF(long long); +} + +TEST(MSGPACK, vrefbuffer_unsigned_short) +{ + GEN_TEST_VREF(unsigned short); +} + +TEST(MSGPACK, vrefbuffer_unsigned_int) +{ + GEN_TEST_VREF(unsigned int); +} + +TEST(MSGPACK, vrefbuffer_unsigned_long) +{ + GEN_TEST_VREF(unsigned long); +} + +TEST(MSGPACK, vrefbuffer_unsigned_long_long) +{ + GEN_TEST_VREF(unsigned long long); +} + +TEST(MSGPACK, vrefbuffer_uint8) +{ + GEN_TEST_VREF(uint8_t); +} + +TEST(MSGPACK, vrefbuffer_uint16) +{ + GEN_TEST_VREF(uint16_t); +} + +TEST(MSGPACK, vrefbuffer_uint32) +{ + GEN_TEST_VREF(uint32_t); +} + +TEST(MSGPACK, vrefbuffer_uint64) +{ + GEN_TEST_VREF(uint64_t); +} + +TEST(MSGPACK, vrefbuffer_int8) +{ + GEN_TEST_VREF(int8_t); +} + +TEST(MSGPACK, vrefbuffer_int16) +{ + GEN_TEST_VREF(int16_t); +} + +TEST(MSGPACK, vrefbuffer_int32) +{ + GEN_TEST_VREF(int32_t); +} + +TEST(MSGPACK, vrefbuffer_int64) +{ + GEN_TEST_VREF(int64_t); +} From 8da7b692f662ef0b65974864bf70d7437070b254 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Sep 2009 13:50:19 +0900 Subject: [PATCH 0300/1172] cpp: define, type::tuple: GENERATION_LIMIT = 31 --- cpp/type/define.hpp.erb | 2 +- cpp/type/tuple.hpp.erb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/type/define.hpp.erb b/cpp/type/define.hpp.erb index d9d3fb1..2eac4f4 100644 --- a/cpp/type/define.hpp.erb +++ b/cpp/type/define.hpp.erb @@ -33,7 +33,7 @@ namespace msgpack { namespace type { -<% GENERATION_LIMIT = 15 %> +<% GENERATION_LIMIT = 31 %> template , typename A<%=i%> = void<%}%>> struct define; diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index d8ddbcd..501a0f1 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -26,7 +26,7 @@ namespace type { // FIXME operator== // FIXME operator!= -<% GENERATION_LIMIT = 15 %> +<% GENERATION_LIMIT = 31 %> template , typename A<%=i%> = void<%}%>> struct tuple; From bf3cb63d468c333d1455d21012aaa679d110af3e Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Sep 2009 13:53:12 +0900 Subject: [PATCH 0301/1172] cpp: version 0.3.8 --- configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.in b/configure.in index de71fc9..c2ca872 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.7) +AM_INIT_AUTOMAKE(msgpack, 0.3.8) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) From d0b76814b026d18a13fe67d99505634f8878250d Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sat, 12 Sep 2009 03:12:53 +0900 Subject: [PATCH 0302/1172] cpp: add tests for stream unpacker api --- cpp/test.cpp | 120 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/cpp/test.cpp b/cpp/test.cpp index 3907c47..4447890 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -670,3 +670,123 @@ TEST(MSGPACK, vrefbuffer_int64) { GEN_TEST_VREF(int64_t); } + +//----------------------------------------------------------------------------- + +#define GEN_TEST_STREAM(test_type) \ + for (unsigned int k = 0; k < kLoop; k++) { \ + msgpack::sbuffer sbuf; \ + msgpack::packer pk(sbuf); \ + typedef std::vector vec_type; \ + vec_type vec; \ + for(unsigned int i = 0; i < rand() % kLoop; ++i) { \ + vec_type::value_type r = rand(); \ + vec.push_back(r); \ + pk.pack(r); \ + } \ + msgpack::unpacker pac; \ + vec_type::const_iterator it = vec.begin(); \ + const char *p = sbuf.data(); \ + const char * const pend = p + sbuf.size(); \ + while (p < pend) { \ + const size_t sz = std::min(pend - p, rand() % 128); \ + pac.reserve_buffer(sz); \ + memcpy(pac.buffer(), p, sz); \ + pac.buffer_consumed(sz); \ + while (pac.execute()) { \ + if (it == vec.end()) goto out; \ + msgpack::object obj = pac.data(); \ + msgpack::zone *life = pac.release_zone(); \ + EXPECT_TRUE(life != NULL); \ + pac.reset(); \ + vec_type::value_type val; \ + obj.convert(&val); \ + EXPECT_EQ(*it, val); \ + ++it; \ + delete life; \ + } \ + p += sz; \ + } \ + out: \ + ; \ + } + +TEST(MSGPACK, stream_short) +{ + GEN_TEST_STREAM(short); +} + +TEST(MSGPACK, stream_int) +{ + GEN_TEST_STREAM(int); +} + +TEST(MSGPACK, stream_long) +{ + GEN_TEST_STREAM(long); +} + +TEST(MSGPACK, stream_long_long) +{ + GEN_TEST_STREAM(long long); +} + +TEST(MSGPACK, stream_unsigned_short) +{ + GEN_TEST_STREAM(unsigned short); +} + +TEST(MSGPACK, stream_unsigned_int) +{ + GEN_TEST_STREAM(unsigned int); +} + +TEST(MSGPACK, stream_unsigned_long) +{ + GEN_TEST_STREAM(unsigned long); +} + +TEST(MSGPACK, stream_unsigned_long_long) +{ + GEN_TEST_STREAM(unsigned long long); +} + +TEST(MSGPACK, stream_uint8) +{ + GEN_TEST_STREAM(uint8_t); +} + +TEST(MSGPACK, stream_uint16) +{ + GEN_TEST_STREAM(uint16_t); +} + +TEST(MSGPACK, stream_uint32) +{ + GEN_TEST_STREAM(uint32_t); +} + +TEST(MSGPACK, stream_uint64) +{ + GEN_TEST_STREAM(uint64_t); +} + +TEST(MSGPACK, stream_int8) +{ + GEN_TEST_STREAM(int8_t); +} + +TEST(MSGPACK, stream_int16) +{ + GEN_TEST_STREAM(int16_t); +} + +TEST(MSGPACK, stream_int32) +{ + GEN_TEST_STREAM(int32_t); +} + +TEST(MSGPACK, stream_int64) +{ + GEN_TEST_STREAM(int64_t); +} From c8ad32a39ed19828fe98c9d6838002653861d3e5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 5 Oct 2009 23:31:08 +0900 Subject: [PATCH 0303/1172] cpp: preprocess.sh: more verbose --- cpp/preprocess.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cpp/preprocess.sh b/cpp/preprocess.sh index 332dd9c..2e06c10 100755 --- a/cpp/preprocess.sh +++ b/cpp/preprocess.sh @@ -1,8 +1,14 @@ #!/bin/sh preprocess() { - erb $1.erb > $1.tmp - mv $1.tmp $1 + ruby -r erb -e 'puts ERB.new(ARGF.read).result' $1.erb > $1.tmp + if [ "$?" != 0 ]; then + echo "" + echo "** preprocess failed **" + echo "" + else + mv $1.tmp $1 + fi } preprocess msgpack/type/tuple.hpp From 3424dc916c27e7e82d73285e468d111449e34460 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 9 Oct 2009 17:49:30 +0900 Subject: [PATCH 0304/1172] cpp: add missing type::tuple::tuple(object o) --- cpp/type/tuple.hpp.erb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index 501a0f1..2930ae0 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -91,6 +91,8 @@ private: template <> struct tuple<> { + tuple() {} + tuple(object o) { o.convert(this); } typedef tuple<> value_type; }; <%0.upto(GENERATION_LIMIT) {|i|%> From c6a2569af8839ce68f96cca1d3aa61c0ac6ed4a6 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 22 Oct 2009 14:32:39 +0900 Subject: [PATCH 0305/1172] - generate README automatically - added LICENSE term --- perl/Makefile.PL | 1 + perl/lib/Data/MessagePack.pm | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index dc74d23..ac83f72 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -1,6 +1,7 @@ use inc::Module::Install; name 'Data-MessagePack'; all_from 'lib/Data/MessagePack.pm'; +readme_from 'lib/Data/MessagePack.pm'; perl_version '5.008005'; license 'perl'; diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 2ab33f1..dde7302 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -44,6 +44,12 @@ Pack the string as int when the value looks like int(EXPERIMENTAL). Tokuhiro Matsuno +=head1 LICENSE + +This library is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + + =head1 SEE ALSO L From 68176e10f5266978a06249049f8c37c7930093c7 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 22 Oct 2009 14:34:53 +0900 Subject: [PATCH 0306/1172] added some pattern to MANIFEST.SKIP --- perl/MANIFEST.SKIP | 2 ++ 1 file changed, 2 insertions(+) diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP index f524216..a318e47 100644 --- a/perl/MANIFEST.SKIP +++ b/perl/MANIFEST.SKIP @@ -21,3 +21,5 @@ \.sw[pon]$ ^\.gitignore$ ppport.h +\.o$ +\.bs$ From 5393a0df16f3bbdf296222b6337db5d57fe7c3a6 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Oct 2009 01:27:09 +0900 Subject: [PATCH 0307/1172] import MessagePack for Java implementation plan 1 --- java-plan1/MessagePack.java | 31 +++ java-plan1/Packer.java | 266 +++++++++++++++++++++++++ java-plan1/Unpacker.java | 385 ++++++++++++++++++++++++++++++++++++ java-plan1/run.sh | 3 + java-plan1/test.java | 27 +++ 5 files changed, 712 insertions(+) create mode 100644 java-plan1/MessagePack.java create mode 100644 java-plan1/Packer.java create mode 100644 java-plan1/Unpacker.java create mode 100755 java-plan1/run.sh create mode 100644 java-plan1/test.java diff --git a/java-plan1/MessagePack.java b/java-plan1/MessagePack.java new file mode 100644 index 0000000..143f6b5 --- /dev/null +++ b/java-plan1/MessagePack.java @@ -0,0 +1,31 @@ +import java.nio.ByteBuffer; +import java.io.InputStream; +import java.io.IOException; + +public class MessagePack { + + static public Object unpack(InputStream source) + { + // FIXME not implemented yet + return null; + } + + static public Object unpack(byte[] source, int len) throws IOException + { + // FIXME not implemented yet + return null; + } + + static public Object unpack(ByteBuffer source) throws IOException + { + // FIXME not implemented yet + return null; + } + + static public Object unpack(ByteBuffer[] source) throws IOException + { + // FIXME not implemented yet + return null; + } +} + diff --git a/java-plan1/Packer.java b/java-plan1/Packer.java new file mode 100644 index 0000000..ea6b2d4 --- /dev/null +++ b/java-plan1/Packer.java @@ -0,0 +1,266 @@ +import java.io.*; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.math.BigInteger; + +public class Packer { + protected byte[] castBytes = new byte[9]; + protected ByteBuffer castBuffer = ByteBuffer.wrap(castBytes); + protected OutputStream out; + + public Packer(OutputStream out) + { + this.out = out; + } + + public Packer packByte(byte d) throws IOException + { + if(d < -(1<<5)) { + castBytes[0] = (byte)0xd1; + castBytes[1] = d; + out.write(castBytes, 0, 2); + } else { + out.write(d); + } + return this; + } + + public Packer packShort(short d) throws IOException + { + if(d < -(1<<5)) { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } + } + return this; + } + + public Packer packInt(int d) throws IOException + { + if(d < -(1<<5)) { + if(d < -(1<<15)) { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } else if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else if(d < (1<<16)) { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } + } + return this; + } + + public Packer packLong(long d) throws IOException + { + if(d < -(1L<<5)) { + if(d < -(1L<<15)) { + if(d < -(1L<<31)) { + // signed 64 + castBytes[0] = (byte)0xd3; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } else { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } + } else { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1L<<16)) { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + //System.out.println("pack uint 16 "+(short)d); + } + } else { + if(d < (1L<<32)) { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } else { + // unsigned 64 + castBytes[0] = (byte)0xcf; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } + } + } + return this; + } + + public Packer packFloat(float d) throws IOException + { + castBytes[0] = (byte)0xca; + castBuffer.putFloat(1, d); + out.write(castBytes, 0, 5); + return this; + } + + public Packer packDouble(double d) throws IOException + { + castBytes[0] = (byte)0xcb; + castBuffer.putDouble(1, d); + out.write(castBytes, 0, 9); + return this; + } + + public Packer packNil() throws IOException + { + out.write((byte)0xc0); + return this; + } + + public Packer packTrue() throws IOException + { + out.write((byte)0xc3); + return this; + } + + public Packer packFalse() throws IOException + { + out.write((byte)0xc2); + return this; + } + + public Packer packArray(int n) throws IOException + { + if(n < 16) { + final int d = 0x90 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xdc; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdd; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packMap(int n) throws IOException + { + if(n < 16) { + final int d = 0x80 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xde; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdf; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRaw(int n) throws IOException + { + if(n < 32) { + final int d = 0xa0 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xda; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdb; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRawBody(byte[] b) throws IOException + { + out.write(b); + return this; + } + + public Packer packRawBody(byte[] b, int off, int length) throws IOException + { + out.write(b, off, length); + return this; + } + + //public Packer pack(Object o) throws IOException +} + diff --git a/java-plan1/Unpacker.java b/java-plan1/Unpacker.java new file mode 100644 index 0000000..be86344 --- /dev/null +++ b/java-plan1/Unpacker.java @@ -0,0 +1,385 @@ +import java.io.*; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.math.BigInteger; + +public class Unpacker { + protected static final int CS_HEADER = 0x00; + protected static final int CS_FLOAT = 0x0a; + protected static final int CS_DOUBLE = 0x0b; + protected static final int CS_UINT_8 = 0x0c; + protected static final int CS_UINT_16 = 0x0d; + protected static final int CS_UINT_32 = 0x0e; + protected static final int CS_UINT_64 = 0x0f; + protected static final int CS_INT_8 = 0x10; + protected static final int CS_INT_16 = 0x11; + protected static final int CS_INT_32 = 0x12; + protected static final int CS_INT_64 = 0x13; + protected static final int CS_RAW_16 = 0x1a; + protected static final int CS_RAW_32 = 0x1b; + protected static final int CS_ARRAY_16 = 0x1c; + protected static final int CS_ARRAY_32 = 0x1d; + protected static final int CS_MAP_16 = 0x1e; + protected static final int CS_MAP_32 = 0x1f; + protected static final int ACS_RAW_VALUE = 0x20; + protected static final int CT_ARRAY_ITEM = 0x00; + protected static final int CT_MAP_KEY = 0x01; + protected static final int CT_MAP_VALUE = 0x02; + + protected static final int MAX_STACK_SIZE = 16; + + protected int cs = CS_HEADER; + protected int trail = 0; + protected int top = -1; + protected boolean finished = false; + protected int[] stack_ct = new int[MAX_STACK_SIZE]; + protected int[] stack_count = new int[MAX_STACK_SIZE]; + protected Object[] stack_obj = new Object[MAX_STACK_SIZE]; + protected Object[] stack_map_key = new Object[MAX_STACK_SIZE]; + //protected Object user; + protected ByteBuffer castBuffer = ByteBuffer.allocate(8); + + public Object getData() + { + return stack_obj[0]; + } + + public boolean isFinished() + { + return finished; + } + + + @SuppressWarnings("unchecked") + public int execute(byte[] src, int off, int length) throws IOException + { + if(off >= length) { return off; } + + int limit = off + length; + int i = off; + int count; + + Object obj = null; + + _out: do { + _header_again: { + //System.out.println("while i:"+i); + + int b = src[i]; + + _push: { + _fixed_trail_again: + if(cs == CS_HEADER) { + + if((b & 0x80) == 0) { // Positive Fixnum + //System.out.println("positive fixnum "+b); + obj = b; + break _push; + } + + if((b & 0xe0) == 0xe0) { // Negative Fixnum + //System.out.println("negative fixnum "+b); + obj = b; + break _push; + } + + if((b & 0xe0) == 0xa0) { // FixRaw + trail = b & 0x1f; + if(trail == 0) { + obj = new byte[0]; + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + } + + if((b & 0xf0) == 0x90) { // FixArray + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + count = b & 0x0f; + ++top; + stack_obj[top] = new ArrayList(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + //System.out.println("fixarray count:"+count); + break _header_again; + } + + if((b & 0xf0) == 0x80) { // FixMap + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + count = b & 0x0f; + ++top; + stack_obj[top] = new HashMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + } + + switch(b & 0xff) { // FIXME + case 0xc0: // nil + obj = null; + break _push; + case 0xc2: // false + obj = false; + break _push; + case 0xc3: // true + obj = true; + break _push; + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + trail = 1 << (b & 0x03); + cs = b & 0x1f; + //System.out.println("a trail "+trail+" cs:"+cs); + break _fixed_trail_again; + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + trail = 2 << (b & 0x01); + cs = b & 0x1f; + //System.out.println("b trail "+trail+" cs:"+cs); + break _fixed_trail_again; + default: + //System.out.println("unknown b "+(b&0xff)); + throw new IOException("parse error"); + } + + } // _fixed_trail_again + + do { + _fixed_trail_again: { + + if(limit - i <= trail) { break _out; } + int n = i + 1; + i += trail; + + switch(cs) { + case CS_FLOAT: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = castBuffer.getFloat(0); + //System.out.println("float "+obj); + break _push; + case CS_DOUBLE: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + obj = castBuffer.getDouble(0); + //System.out.println("double "+obj); + break _push; + case CS_UINT_8: + //System.out.println(n); + //System.out.println(src[n]); + //System.out.println(src[n+1]); + //System.out.println(src[n-1]); + obj = ((int)src[n]) & 0xff; + //System.out.println("uint8 "+obj); + break _push; + case CS_UINT_16: + //System.out.println(src[n]); + //System.out.println(src[n+1]); + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = ((int)castBuffer.getShort(0)) & 0xffff; + //System.out.println("uint 16 "+obj); + break _push; + case CS_UINT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + { + // FIXME always long? + int o = castBuffer.getInt(0); + if(o < 0) { + obj = ((long)o) & 0xffffffffL; + } else { + obj = o; + } + } + //System.out.println("uint 32 "+obj); + break _push; + case CS_UINT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + { + // FIXME always BigInteger? + long o = castBuffer.getLong(0); + if(o < 0) { + obj = BigInteger.valueOf(o & 0x7fffffffL).setBit(31); + } else { + obj = o; + } + } + throw new IOException("uint 64 is not supported"); + case CS_INT_8: + obj = (int)src[n]; + break _push; + case CS_INT_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = (int)castBuffer.getShort(0); + break _push; + case CS_INT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = (int)castBuffer.getInt(0); + break _push; + case CS_INT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + { + // FIXME always long? + long o = castBuffer.getLong(0); + if(o <= 0x7fffffffL && o > -0x80000000L) { + obj = (int)o; + } else { + obj = o; + } + } + //System.out.println("long "+obj); + break _push; + case CS_RAW_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + trail = ((int)castBuffer.getShort(0)) & 0xffff; + if(trail == 0) { + obj = new byte[0]; + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + case CS_RAW_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + trail = castBuffer.getInt(0) & 0x7fffffff; + if(trail == 0) { + obj = new byte[0]; + break _push; + } + cs = ACS_RAW_VALUE; + case ACS_RAW_VALUE: + obj = ByteBuffer.wrap(src, n, trail); // FIXME プール? コピー + break _push; + case CS_ARRAY_16: + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + ++top; + stack_obj[top] = new ArrayList(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + break _header_again; + case CS_ARRAY_32: + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + ++top; + stack_obj[top] = new ArrayList(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + break _header_again; + case CS_MAP_16: + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + ++top; + stack_obj[top] = new HashMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + case CS_MAP_32: + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + ++top; + stack_obj[top] = new HashMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + default: + throw new IOException("parse error"); + } + + } // _fixed_trail_again + } while(true); + } // _push + + do { + _push: { + //System.out.println("push top:"+top); + if(top == -1) { + stack_obj[0] = obj; + finished = true; + break _out; + } + + switch(stack_ct[top]) { + case CT_ARRAY_ITEM: + //System.out.println("array item "+obj); + ((ArrayList)(stack_obj[top])).add(obj); + if(--stack_count[top] == 0) { + obj = stack_obj[top]; + --top; + break _push; + } + break _header_again; + case CT_MAP_KEY: + //System.out.println("map key:"+top+" "+obj); + stack_map_key[top] = obj; + stack_ct[top] = CT_MAP_VALUE; + break _header_again; + case CT_MAP_VALUE: + //System.out.println("map value:"+top+" "+obj); + ((HashMap)(stack_obj[top])).put(stack_map_key[top], obj); + if(--stack_count[top] == 0) { + obj = stack_obj[top]; + --top; + break _push; + } + stack_ct[top] = CT_MAP_KEY; + break _header_again; + default: + throw new IOException("parse error"); + } + } // _push + } while(true); + + } // _header_again + cs = CS_HEADER; + ++i; + } while(i < limit); // _out + + return i - off; + } +} + diff --git a/java-plan1/run.sh b/java-plan1/run.sh new file mode 100755 index 0000000..1539a37 --- /dev/null +++ b/java-plan1/run.sh @@ -0,0 +1,3 @@ +#!/bin/sh +javac MessagePack.java Packer.java Unpacker.java test.java +java test diff --git a/java-plan1/test.java b/java-plan1/test.java new file mode 100644 index 0000000..938a687 --- /dev/null +++ b/java-plan1/test.java @@ -0,0 +1,27 @@ +import java.util.*; +import java.io.*; + +class OpenByteArrayOutputStream extends ByteArrayOutputStream { + int getCount() { return count; } + byte[] getBuffer() { return buf; } +} + +public class test { + public static void main(String[] args) throws IOException + { + OpenByteArrayOutputStream out = new OpenByteArrayOutputStream(); + + Packer pk = new Packer(out); + pk.packArray(3) + .packInt(0) + .packByte((byte)1) + .packDouble(0.1); + + Unpacker pac = new Unpacker(); + int nlen = pac.execute(out.getBuffer(), 0, out.getCount()); + if(pac.isFinished()) { + System.out.println(pac.getData()); + } + } +} + From d39c016e1d84765a50ae3c7cbfc3d6869faa9c07 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Oct 2009 02:41:55 +0900 Subject: [PATCH 0308/1172] java: fix streaming de/serializer --- java-plan1/Unpacker.java | 22 ++++++++++-- java-plan1/test.java | 75 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/java-plan1/Unpacker.java b/java-plan1/Unpacker.java index be86344..32945de 100644 --- a/java-plan1/Unpacker.java +++ b/java-plan1/Unpacker.java @@ -50,13 +50,26 @@ public class Unpacker { return finished; } + public void reset() + { + for(int i=0; i <= top; ++top) { + stack_ct[top] = 0; + stack_count[top] = 0; + stack_obj[top] = null; + stack_map_key[top] = null; + } + cs = CS_HEADER; + trail = 0; + top = -1; + finished = false; + } @SuppressWarnings("unchecked") public int execute(byte[] src, int off, int length) throws IOException { if(off >= length) { return off; } - int limit = off + length; + int limit = length; int i = off; int count; @@ -64,7 +77,7 @@ public class Unpacker { _out: do { _header_again: { - //System.out.println("while i:"+i); + //System.out.println("while i:"+i+" limit:"+limit); int b = src[i]; @@ -349,6 +362,7 @@ public class Unpacker { ((ArrayList)(stack_obj[top])).add(obj); if(--stack_count[top] == 0) { obj = stack_obj[top]; + stack_obj[top] = null; --top; break _push; } @@ -363,6 +377,8 @@ public class Unpacker { ((HashMap)(stack_obj[top])).put(stack_map_key[top], obj); if(--stack_count[top] == 0) { obj = stack_obj[top]; + stack_map_key[top] = null; + stack_obj[top] = null; --top; break _push; } @@ -379,7 +395,7 @@ public class Unpacker { ++i; } while(i < limit); // _out - return i - off; + return i; } } diff --git a/java-plan1/test.java b/java-plan1/test.java index 938a687..5b2349f 100644 --- a/java-plan1/test.java +++ b/java-plan1/test.java @@ -6,21 +6,88 @@ class OpenByteArrayOutputStream extends ByteArrayOutputStream { byte[] getBuffer() { return buf; } } + public class test { + public static void main(String[] args) throws IOException + { + testSimple(); + testStreaming(); + } + + + public static void testSimple() throws IOException { OpenByteArrayOutputStream out = new OpenByteArrayOutputStream(); Packer pk = new Packer(out); pk.packArray(3) - .packInt(0) - .packByte((byte)1) - .packDouble(0.1); + .packInt(1) + .packByte((byte)2) + .packDouble(0.3); Unpacker pac = new Unpacker(); int nlen = pac.execute(out.getBuffer(), 0, out.getCount()); + if(pac.isFinished()) { - System.out.println(pac.getData()); + System.out.println("testSimple: "+pac.getData()); + } + } + + + public static void testStreaming() throws IOException + { + OpenByteArrayOutputStream out = new OpenByteArrayOutputStream(); + + //// + // sender + // + // initialize the streaming serializer + Packer pk = new Packer(out); + + // serialize 2 objects + pk.packArray(3) + .packInt(0) + .packByte((byte)1) + .packDouble(0.2); + pk.packArray(3) + .packInt(3) + .packByte((byte)4) + .packDouble(0.5); + + // send it through the network + InputStream sock = new ByteArrayInputStream(out.getBuffer(), 0, out.getCount()); + + + //// + // receiver + // + // initialize the streaming deserializer + Unpacker pac = new Unpacker(); + int parsed = 0; + + byte[] buf = new byte[1024]; + int buflen = 0; + + while(true) { + // receive data from the network + int c = sock.read(); + if(c < 0) { return; } + + buf[buflen++] = (byte)c; + + // deserialize + parsed = pac.execute(buf, parsed, buflen); + if(pac.isFinished()) { + // get an object + Object msg = pac.getData(); + System.out.println("testStreaming: "+msg); + + // reset the streaming deserializer + pac.reset(); + buflen = 0; + parsed = 0; + } } } } From 4758d9f04b115bbccb568e262e49d53c5549edb5 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 5 Nov 2009 14:24:47 +0900 Subject: [PATCH 0309/1172] Fix to use MANIEFST.in. --- python/MANIFEST | 9 --------- python/MANIFEST.in | 2 ++ 2 files changed, 2 insertions(+), 9 deletions(-) delete mode 100644 python/MANIFEST create mode 100644 python/MANIFEST.in diff --git a/python/MANIFEST b/python/MANIFEST deleted file mode 100644 index f6c43b0..0000000 --- a/python/MANIFEST +++ /dev/null @@ -1,9 +0,0 @@ -setup.py -msgpack/pack.h -msgpack/unpack.h -msgpack/_msgpack.pyx -msgpack/__init__.py -msgpack/pack_define.h -msgpack/pack_template.h -msgpack/unpack_define.h -msgpack/unpack_template.h diff --git a/python/MANIFEST.in b/python/MANIFEST.in new file mode 100644 index 0000000..6841ffe --- /dev/null +++ b/python/MANIFEST.in @@ -0,0 +1,2 @@ +include setup.py +recursive-include msgpack *.h *.c *.pyx From fb7f2ed4ada6188f0ea5e0b3e1b88a6eccc3457a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 5 Nov 2009 14:24:47 +0900 Subject: [PATCH 0310/1172] Fix to use MANIEFST.in. --- MANIFEST | 9 --------- MANIFEST.in | 2 ++ 2 files changed, 2 insertions(+), 9 deletions(-) delete mode 100644 MANIFEST create mode 100644 MANIFEST.in diff --git a/MANIFEST b/MANIFEST deleted file mode 100644 index f6c43b0..0000000 --- a/MANIFEST +++ /dev/null @@ -1,9 +0,0 @@ -setup.py -msgpack/pack.h -msgpack/unpack.h -msgpack/_msgpack.pyx -msgpack/__init__.py -msgpack/pack_define.h -msgpack/pack_template.h -msgpack/unpack_define.h -msgpack/unpack_template.h diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..6841ffe --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include setup.py +recursive-include msgpack *.h *.c *.pyx From 93a95725fc45d7d0047578ecdc5b2ae4e900970f Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 5 Nov 2009 14:26:12 +0900 Subject: [PATCH 0311/1172] Fix Makefile --- python/Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/Makefile b/python/Makefile index d90789a..267415c 100644 --- a/python/Makefile +++ b/python/Makefile @@ -1,7 +1,6 @@ all: - python setup.py build_ext -i -f - python setup.py build - python setup.py sdist + python setup_dev.py build_ext -i -f + python setup.py build sdist .PHONY: test test: From e72aa01fe5192960618544ec0394112904b24c7d Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 5 Nov 2009 14:26:12 +0900 Subject: [PATCH 0312/1172] Fix Makefile --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d90789a..267415c 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ all: - python setup.py build_ext -i -f - python setup.py build - python setup.py sdist + python setup_dev.py build_ext -i -f + python setup.py build sdist .PHONY: test test: From e39e1d4f602b0202b830f8e672e2116bdb8b9f34 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 12 Nov 2009 01:10:36 +0900 Subject: [PATCH 0313/1172] import MessagePack for Java implementation plan 2 --- java-plan1/MessagePack.java | 6 + java-plan2/build.xml | 15 + java-plan2/src/org/msgpack/GenericArray.java | 25 ++ .../src/org/msgpack/GenericBoolean.java | 17 + java-plan2/src/org/msgpack/GenericLong.java | 17 + java-plan2/src/org/msgpack/GenericMap.java | 25 ++ java-plan2/src/org/msgpack/GenericObject.java | 52 +++ java-plan2/src/org/msgpack/GenericRaw.java | 59 +++ java-plan2/src/org/msgpack/GenericRawRef.java | 55 +++ java-plan2/src/org/msgpack/GenericShort.java | 29 ++ .../src/org/msgpack/MessageConvertable.java | 7 + .../src/org/msgpack/MessageMergeable.java | 7 + .../src/org/msgpack/MessagePackException.java | 10 + .../src/org/msgpack/MessagePackable.java | 9 + java-plan2/src/org/msgpack/Packer.java | 332 +++++++++++++++ .../src/org/msgpack/UnbufferedUnpacker.java | 74 ++++ .../src/org/msgpack/UnpackException.java | 35 ++ .../src/org/msgpack/UnpackIterator.java | 53 +++ java-plan2/src/org/msgpack/Unpacker.java | 269 ++++++++++++ .../src/org/msgpack/impl/ArrayBuilder.java | 7 + .../msgpack/impl/GenericObjectBuilder.java | 127 ++++++ .../impl/GenericZeroCopyObjectBuilder.java | 12 + .../src/org/msgpack/impl/MapBuilder.java | 8 + .../src/org/msgpack/impl/ObjectBuilder.java | 16 + .../msgpack/impl/SpecificObjectBuilder.java | 203 +++++++++ .../src/org/msgpack/impl/UnpackerImpl.java | 395 ++++++++++++++++++ .../src/org/msgpack/schema/ArraySchema.java | 61 +++ .../org/msgpack/schema/ClassGenerator.java | 230 ++++++++++ .../src/org/msgpack/schema/ClassSchema.java | 39 ++ .../src/org/msgpack/schema/FieldSchema.java | 31 ++ .../msgpack/schema/GenericClassSchema.java | 89 ++++ .../msgpack/schema/GenericFieldSchema.java | 25 ++ .../src/org/msgpack/schema/IntSchema.java | 58 +++ .../src/org/msgpack/schema/LongSchema.java | 58 +++ .../src/org/msgpack/schema/MapSchema.java | 70 ++++ .../src/org/msgpack/schema/ObjectSchema.java | 33 ++ .../org/msgpack/schema/PrimitiveSchema.java | 21 + .../src/org/msgpack/schema/RawSchema.java | 44 ++ .../src/org/msgpack/schema/SSchemaParser.java | 246 +++++++++++ java-plan2/src/org/msgpack/schema/Schema.java | 93 +++++ .../src/org/msgpack/schema/ShortSchema.java | 46 ++ .../msgpack/schema/SpecificClassSchema.java | 156 +++++++ .../msgpack/schema/SpecificFieldSchema.java | 78 ++++ .../src/org/msgpack/schema/StringSchema.java | 48 +++ java-plan2/test/Generate.java | 18 + .../src/serializers/msgpack/MediaContent.java | 277 ++++++++++++ .../serializers/msgpack/MediaContent.mpacs | 21 + .../msgpack/MessagePackSerializer.java | 70 ++++ 48 files changed, 3676 insertions(+) create mode 100644 java-plan2/build.xml create mode 100644 java-plan2/src/org/msgpack/GenericArray.java create mode 100644 java-plan2/src/org/msgpack/GenericBoolean.java create mode 100644 java-plan2/src/org/msgpack/GenericLong.java create mode 100644 java-plan2/src/org/msgpack/GenericMap.java create mode 100644 java-plan2/src/org/msgpack/GenericObject.java create mode 100644 java-plan2/src/org/msgpack/GenericRaw.java create mode 100644 java-plan2/src/org/msgpack/GenericRawRef.java create mode 100644 java-plan2/src/org/msgpack/GenericShort.java create mode 100644 java-plan2/src/org/msgpack/MessageConvertable.java create mode 100644 java-plan2/src/org/msgpack/MessageMergeable.java create mode 100644 java-plan2/src/org/msgpack/MessagePackException.java create mode 100644 java-plan2/src/org/msgpack/MessagePackable.java create mode 100644 java-plan2/src/org/msgpack/Packer.java create mode 100644 java-plan2/src/org/msgpack/UnbufferedUnpacker.java create mode 100644 java-plan2/src/org/msgpack/UnpackException.java create mode 100644 java-plan2/src/org/msgpack/UnpackIterator.java create mode 100644 java-plan2/src/org/msgpack/Unpacker.java create mode 100644 java-plan2/src/org/msgpack/impl/ArrayBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/MapBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/ObjectBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/UnpackerImpl.java create mode 100644 java-plan2/src/org/msgpack/schema/ArraySchema.java create mode 100644 java-plan2/src/org/msgpack/schema/ClassGenerator.java create mode 100644 java-plan2/src/org/msgpack/schema/ClassSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/FieldSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/GenericClassSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/GenericFieldSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/IntSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/LongSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/MapSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/ObjectSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/PrimitiveSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/RawSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/SSchemaParser.java create mode 100644 java-plan2/src/org/msgpack/schema/Schema.java create mode 100644 java-plan2/src/org/msgpack/schema/ShortSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/SpecificClassSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/StringSchema.java create mode 100644 java-plan2/test/Generate.java create mode 100644 java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java create mode 100644 java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs create mode 100644 java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java diff --git a/java-plan1/MessagePack.java b/java-plan1/MessagePack.java index 143f6b5..0a4f442 100644 --- a/java-plan1/MessagePack.java +++ b/java-plan1/MessagePack.java @@ -29,3 +29,9 @@ public class MessagePack { } } +/* +public interface MessagePackable { + public void toMessagePack(Packer pk); +} +*/ + diff --git a/java-plan2/build.xml b/java-plan2/build.xml new file mode 100644 index 0000000..d382ce0 --- /dev/null +++ b/java-plan2/build.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/java-plan2/src/org/msgpack/GenericArray.java b/java-plan2/src/org/msgpack/GenericArray.java new file mode 100644 index 0000000..53799ca --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericArray.java @@ -0,0 +1,25 @@ +package org.msgpack; + +import java.util.List; +import java.util.ArrayList; + +public class GenericArray extends GenericObject { + private ArrayList array; + + public GenericArray(int length) + { + this.array = new ArrayList(length); + } + + public void add(GenericObject element) + { + array.add(element); + } + + @Override + public List asArray() + { + return array; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericBoolean.java b/java-plan2/src/org/msgpack/GenericBoolean.java new file mode 100644 index 0000000..908128f --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericBoolean.java @@ -0,0 +1,17 @@ +package org.msgpack; + +public class GenericBoolean extends GenericObject { + boolean value; + + public GenericBoolean(boolean value) + { + this.value = value; + } + + @Override + public boolean asBoolean() + { + return value; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericLong.java b/java-plan2/src/org/msgpack/GenericLong.java new file mode 100644 index 0000000..7cc9110 --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericLong.java @@ -0,0 +1,17 @@ +package org.msgpack; + +public class GenericLong extends GenericObject { + long value; + + public GenericLong(long value) + { + this.value = value; + } + + @Override + public long asLong() + { + return value; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericMap.java b/java-plan2/src/org/msgpack/GenericMap.java new file mode 100644 index 0000000..0a5512a --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericMap.java @@ -0,0 +1,25 @@ +package org.msgpack; + +import java.util.Map; +import java.util.HashMap; + +public class GenericMap extends GenericObject { + private HashMap map; + + public GenericMap(int length) + { + this.map = new HashMap(length); + } + + public void put(GenericObject key, GenericObject value) + { + map.put(key, value); + } + + @Override + public Map asMap() + { + return map; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericObject.java b/java-plan2/src/org/msgpack/GenericObject.java new file mode 100644 index 0000000..32eacd3 --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericObject.java @@ -0,0 +1,52 @@ +package org.msgpack; + +import java.util.List; +import java.util.Map; + +public abstract class GenericObject { + public boolean asBoolean() + { + throw new RuntimeException("type error"); + } + + public byte asByte() + { + throw new RuntimeException("type error"); + } + + public short asShort() + { + throw new RuntimeException("type error"); + } + + public int asInt() + { + throw new RuntimeException("type error"); + } + + public long asLong() + { + throw new RuntimeException("type error"); + } + + public String asString() + { + throw new RuntimeException("type error"); + } + + public byte[] asBytes() + { + throw new RuntimeException("type error"); + } + + public List asArray() + { + throw new RuntimeException("type error"); + } + + public Map asMap() + { + throw new RuntimeException("type error"); + } +} + diff --git a/java-plan2/src/org/msgpack/GenericRaw.java b/java-plan2/src/org/msgpack/GenericRaw.java new file mode 100644 index 0000000..6de03b2 --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericRaw.java @@ -0,0 +1,59 @@ +package org.msgpack; + +import java.nio.charset.Charset; + +public class GenericRaw extends GenericObject { + byte[] bytes; + String string; + + public GenericRaw() + { + this.bytes = new byte[0]; + this.string = null; + } + + public GenericRaw(byte[] bytes) + { + this.bytes = bytes; + this.string = null; + } + + public GenericRaw(String string) + { + this.bytes = null; + this.string = string; + } + + public synchronized void setString(String string) + { + this.string = string; + this.bytes = null; + } + + public synchronized void setBytes(byte[] bytes) + { + this.bytes = bytes; + this.string = null; + } + + private static Charset UTF8_CHARSET = Charset.forName("UTF-8"); + + @Override + public synchronized String asString() + { + if(string == null) { + return string = new String(bytes, UTF8_CHARSET); + } + return string; + } + + @Override + public synchronized byte[] asBytes() + { + if(bytes == null) { + return bytes = string.getBytes(UTF8_CHARSET); + } + return bytes; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericRawRef.java b/java-plan2/src/org/msgpack/GenericRawRef.java new file mode 100644 index 0000000..7007810 --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericRawRef.java @@ -0,0 +1,55 @@ +package org.msgpack; + +import java.nio.charset.Charset; + +public class GenericRawRef extends GenericRaw { + int offset; + int length; + + public GenericRawRef(byte[] bytes, int offset, int length) + { + this.bytes = bytes; + this.offset = offset; + this.length = length; + this.string = null; + } + + public GenericRawRef(String string) + { + this.bytes = null; + this.string = string; + } + + public synchronized void setString(String string) + { + this.string = string; + this.bytes = null; + } + + public synchronized void setBytes(byte[] bytes) + { + this.bytes = bytes; + this.string = null; + } + + private static Charset UTF8_CHARSET = Charset.forName("UTF-8"); + + @Override + public synchronized String asString() + { + if(string == null) { + return string = new String(bytes, UTF8_CHARSET); + } + return string; + } + + @Override + public synchronized byte[] asBytes() + { + if(bytes == null) { + return bytes = string.getBytes(UTF8_CHARSET); + } + return bytes; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericShort.java b/java-plan2/src/org/msgpack/GenericShort.java new file mode 100644 index 0000000..eb2463e --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericShort.java @@ -0,0 +1,29 @@ +package org.msgpack; + +public class GenericShort extends GenericObject { + short value; + + public GenericShort(short value) + { + this.value = value; + } + + @Override + public short asShort() + { + return value; + } + + @Override + public int asInt() + { + return value; + } + + @Override + public long asLong() + { + return value; + } +} + diff --git a/java-plan2/src/org/msgpack/MessageConvertable.java b/java-plan2/src/org/msgpack/MessageConvertable.java new file mode 100644 index 0000000..68c123e --- /dev/null +++ b/java-plan2/src/org/msgpack/MessageConvertable.java @@ -0,0 +1,7 @@ +package org.msgpack; + +public interface MessageConvertable +{ + public void messageConvert(GenericObject obj); +} + diff --git a/java-plan2/src/org/msgpack/MessageMergeable.java b/java-plan2/src/org/msgpack/MessageMergeable.java new file mode 100644 index 0000000..dc50749 --- /dev/null +++ b/java-plan2/src/org/msgpack/MessageMergeable.java @@ -0,0 +1,7 @@ +package org.msgpack; + +public interface MessageMergeable { + public void setField(int index, Object value); + public Object getField(int index); +} + diff --git a/java-plan2/src/org/msgpack/MessagePackException.java b/java-plan2/src/org/msgpack/MessagePackException.java new file mode 100644 index 0000000..dc37989 --- /dev/null +++ b/java-plan2/src/org/msgpack/MessagePackException.java @@ -0,0 +1,10 @@ +package org.msgpack; + +public class MessagePackException extends RuntimeException +{ + public MessagePackException(String message) + { + super(message); + } +} + diff --git a/java-plan2/src/org/msgpack/MessagePackable.java b/java-plan2/src/org/msgpack/MessagePackable.java new file mode 100644 index 0000000..1efff3f --- /dev/null +++ b/java-plan2/src/org/msgpack/MessagePackable.java @@ -0,0 +1,9 @@ +package org.msgpack; + +import java.io.IOException; + +public interface MessagePackable +{ + public void messagePack(Packer pk) throws IOException; +} + diff --git a/java-plan2/src/org/msgpack/Packer.java b/java-plan2/src/org/msgpack/Packer.java new file mode 100644 index 0000000..4545849 --- /dev/null +++ b/java-plan2/src/org/msgpack/Packer.java @@ -0,0 +1,332 @@ +package org.msgpack; + +import java.io.OutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Map; + +public class Packer { + protected byte[] castBytes = new byte[9]; + protected ByteBuffer castBuffer = ByteBuffer.wrap(castBytes); + protected OutputStream out; + + public Packer(OutputStream out) + { + this.out = out; + } + + public Packer packByte(byte d) throws IOException + { + if(d < -(1<<5)) { + castBytes[0] = (byte)0xd1; + castBytes[1] = d; + out.write(castBytes, 0, 2); + } else { + out.write(d); + } + return this; + } + + public Packer packShort(short d) throws IOException + { + if(d < -(1<<5)) { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } + } + return this; + } + + public Packer packInt(int d) throws IOException + { + if(d < -(1<<5)) { + if(d < -(1<<15)) { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } else if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else if(d < (1<<16)) { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } + } + return this; + } + + public Packer packLong(long d) throws IOException + { + if(d < -(1L<<5)) { + if(d < -(1L<<15)) { + if(d < -(1L<<31)) { + // signed 64 + castBytes[0] = (byte)0xd3; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } else { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } + } else { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1L<<16)) { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + //System.out.println("pack uint 16 "+(short)d); + } + } else { + if(d < (1L<<32)) { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } else { + // unsigned 64 + castBytes[0] = (byte)0xcf; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } + } + } + return this; + } + + public Packer packFloat(float d) throws IOException + { + castBytes[0] = (byte)0xca; + castBuffer.putFloat(1, d); + out.write(castBytes, 0, 5); + return this; + } + + public Packer packDouble(double d) throws IOException + { + castBytes[0] = (byte)0xcb; + castBuffer.putDouble(1, d); + out.write(castBytes, 0, 9); + return this; + } + + public Packer packNil() throws IOException + { + out.write((byte)0xc0); + return this; + } + + public Packer packTrue() throws IOException + { + out.write((byte)0xc3); + return this; + } + + public Packer packFalse() throws IOException + { + out.write((byte)0xc2); + return this; + } + + public Packer packArray(int n) throws IOException + { + if(n < 16) { + final int d = 0x90 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xdc; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdd; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packMap(int n) throws IOException + { + if(n < 16) { + final int d = 0x80 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xde; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdf; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRaw(int n) throws IOException + { + if(n < 32) { + final int d = 0xa0 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xda; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdb; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRawBody(byte[] b) throws IOException + { + out.write(b); + return this; + } + + public Packer packRawBody(byte[] b, int off, int length) throws IOException + { + out.write(b, off, length); + return this; + } + + private static Charset UTF8_CHARSET = Charset.forName("UTF-8"); + + public Packer packString(String s) throws IOException + { + byte[] b = ((String)s).getBytes(UTF8_CHARSET); + packRaw(b.length); + packRawBody(b); + return this; + } + + public Packer pack(MessagePackable o) throws IOException + { + o.messagePack(this); + return this; + } + + public Packer pack(Object o) throws IOException + { + if(o == null) { + packNil(); + } else if(o instanceof String) { + byte[] b = ((String)o).getBytes(UTF8_CHARSET); + packRaw(b.length); + packRawBody(b); + } else if(o instanceof byte[]) { + byte[] b = (byte[])o; + packRaw(b.length); + packRawBody(b); + } else if(o instanceof List) { + List l = (List)o; + packArray(l.size()); + for(Object i : l) { pack(i); } + } else if(o instanceof Map) { + Map m = (Map)o; + packMap(m.size()); + for(Map.Entry e : m.entrySet()) { + pack(e.getKey()); + pack(e.getValue()); + } + } else if(o instanceof Boolean) { + if((Boolean)o) { + packTrue(); + } else { + packFalse(); + } + } else if(o instanceof Integer) { + packInt((Integer)o); + } else if(o instanceof Long) { + packLong((Long)o); + } else if(o instanceof Short) { + packShort((Short)o); + } else if(o instanceof Byte) { + packByte((Byte)o); + } else if(o instanceof Float) { + packFloat((Float)o); + } else if(o instanceof Double) { + packDouble((Double)o); + } else if(o instanceof MessagePackable) { + ((MessagePackable)o).messagePack(this); + } else { + throw new IOException("unknown object "+o+" ("+o.getClass()+")"); + } + return this; + } +} + diff --git a/java-plan2/src/org/msgpack/UnbufferedUnpacker.java b/java-plan2/src/org/msgpack/UnbufferedUnpacker.java new file mode 100644 index 0000000..5b5eb97 --- /dev/null +++ b/java-plan2/src/org/msgpack/UnbufferedUnpacker.java @@ -0,0 +1,74 @@ +package org.msgpack; + +import java.lang.Iterable; +import java.io.InputStream; +import java.io.IOException; +import java.util.Iterator; +import org.msgpack.impl.*; +import org.msgpack.schema.Schema; + +public class UnbufferedUnpacker extends UnpackerImpl { + private int offset; + private boolean finished; + private Object data; + + public UnbufferedUnpacker() + { + super(new GenericObjectBuilder()); + this.offset = 0; + this.finished = false; + } + + public UnbufferedUnpacker useSchema(Schema s) + { + super.setBuilder(new SpecificObjectBuilder(s)); + return this; + } + + public Object getData() + { + return data; + } + + public boolean isFinished() + { + return finished; + } + + public void reset() + { + super.reset(); + this.offset = 0; + } + + int getOffset() + { + return offset; + } + + void setOffset(int offset) + { + this.offset = offset; + } + + public int execute(byte[] buffer) throws UnpackException + { + return execute(buffer, 0, buffer.length); + } + + // FIXME + public int execute(byte[] buffer, int offset, int length) throws UnpackException + { + int noffset = super.execute(buffer, offset + this.offset, length); + this.offset = noffset - offset; + if(super.isFinished()) { + this.data = super.getData(); + this.finished = true; + super.reset(); + } else { + this.finished = false; + } + return noffset; + } +} + diff --git a/java-plan2/src/org/msgpack/UnpackException.java b/java-plan2/src/org/msgpack/UnpackException.java new file mode 100644 index 0000000..2081faf --- /dev/null +++ b/java-plan2/src/org/msgpack/UnpackException.java @@ -0,0 +1,35 @@ +package org.msgpack; + +public class UnpackException extends MessagePackException +{ + public static final int EXTRA_BYTES = 1; + public static final int INSUFFICIENT_BYTES = 0; + public static final int PARSE_ERROR = -1; + + private int errorCode; + private Object data; + + public UnpackException(String message, int errorCode) + { + super(message); + this.errorCode = errorCode; + } + + public UnpackException(String message, int errorCode, Object data) + { + super(message); + this.errorCode = errorCode; + this.data = data; + } + + public int getErrorCode() + { + return errorCode; + } + + public Object getData() + { + return data; + } +} + diff --git a/java-plan2/src/org/msgpack/UnpackIterator.java b/java-plan2/src/org/msgpack/UnpackIterator.java new file mode 100644 index 0000000..9010101 --- /dev/null +++ b/java-plan2/src/org/msgpack/UnpackIterator.java @@ -0,0 +1,53 @@ +package org.msgpack; + +import java.io.IOException; +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class UnpackIterator implements Iterator { + private Unpacker pac; + private boolean have; + private Object data; + + UnpackIterator(Unpacker pac) + { + this.pac = pac; + this.have = false; + } + + public boolean hasNext() + { + if(have) { return true; } + try { + while(true) { + if(pac.execute()) { + data = pac.getData(); + pac.reset(); + have = true; + return true; + } + + if(!pac.fill()) { + return false; + } + } + } catch (IOException e) { + return false; + } + } + + public Object next() + { + if(!have) { + throw new NoSuchElementException(); + } + have = false; + return data; + } + + public void remove() + { + throw new UnsupportedOperationException(); + } +} + diff --git a/java-plan2/src/org/msgpack/Unpacker.java b/java-plan2/src/org/msgpack/Unpacker.java new file mode 100644 index 0000000..5a58ce8 --- /dev/null +++ b/java-plan2/src/org/msgpack/Unpacker.java @@ -0,0 +1,269 @@ +package org.msgpack; + +import java.lang.Iterable; +import java.io.InputStream; +import java.io.IOException; +import java.util.Iterator; +import org.msgpack.impl.*; +import org.msgpack.schema.Schema; + +public class Unpacker extends UnpackerImpl implements Iterable { + + public static final int DEFAULT_BUFFER_SIZE = 32*1024; + + private int used; + private int offset; + private int parsed; + private byte[] buffer; + private int bufferReserveSize; + private InputStream stream; + + public Unpacker() + { + super(new GenericZeroCopyObjectBuilder()); + this.used = 0; + this.offset = 0; + this.parsed = 0; + this.buffer = new byte[DEFAULT_BUFFER_SIZE]; + this.bufferReserveSize = DEFAULT_BUFFER_SIZE/2; + this.stream = null; + } + + public Unpacker(int bufferReserveSize) + { + super(new GenericZeroCopyObjectBuilder()); + this.used = 0; + this.offset = 0; + this.parsed = 0; + this.buffer = new byte[bufferReserveSize]; + this.bufferReserveSize = bufferReserveSize/2; + this.stream = null; + } + + public Unpacker(InputStream stream) + { + super(new GenericZeroCopyObjectBuilder()); + this.used = 0; + this.offset = 0; + this.parsed = 0; + this.buffer = new byte[DEFAULT_BUFFER_SIZE]; + this.bufferReserveSize = DEFAULT_BUFFER_SIZE/2; + this.stream = stream; + } + + public Unpacker(InputStream stream, int bufferReserveSize) + { + super(new GenericZeroCopyObjectBuilder()); + this.used = 0; + this.offset = 0; + this.parsed = 0; + this.buffer = new byte[bufferReserveSize]; + this.bufferReserveSize = bufferReserveSize/2; + this.stream = stream; + } + + public Unpacker useSchema(Schema s) + { + super.setBuilder(new SpecificObjectBuilder(s)); + return this; + } + + public void reserveBuffer(int size) + { + if(buffer.length - used >= size) { + return; + } + /* + if(used == parsed && buffer.length >= size) { + // rewind buffer + used = 0; + offset = 0; + return; + } + */ + + int nextSize = buffer.length * 2; + while(nextSize < size + used) { + nextSize *= 2; + } + + byte[] tmp = new byte[nextSize]; + System.arraycopy(buffer, offset, tmp, 0, used - offset); + + buffer = tmp; + used -= offset; + offset = 0; + } + + public byte[] getBuffer() + { + return buffer; + } + + public int getBufferOffset() + { + return used; + } + + public int getBufferCapacity() + { + return buffer.length - used; + } + + public void bufferConsumed(int size) + { + used += size; + } + + public void feed(byte[] buffer) + { + feed(buffer, 0, buffer.length); + } + + public void feed(byte[] buffer, int offset, int length) + { + reserveBuffer(length); + System.arraycopy(buffer, offset, this.buffer, this.offset, length); + bufferConsumed(length); + } + + public boolean fill() throws IOException + { + if(stream == null) { + return false; + } + reserveBuffer(bufferReserveSize); + int rl = stream.read(getBuffer(), getBufferOffset(), getBufferCapacity()); + if(rl <= 0) { + return false; + } + bufferConsumed(rl); + return true; + } + + public Iterator iterator() + { + return new UnpackIterator(this); + } + + public boolean execute() throws UnpackException + { + int noffset = super.execute(buffer, offset, used); + if(noffset <= offset) { + return false; + } + parsed += noffset - offset; + offset = noffset; + return super.isFinished(); + } + + public Object getData() + { + return super.getData(); + } + + public void reset() + { + super.reset(); + parsed = 0; + } + + public int getMessageSize() + { + return parsed - offset + used; + } + + public int getParsedSize() + { + return parsed; + } + + public int getNonParsedSize() + { + return used - offset; + } + + public void skipNonparsedBuffer(int size) + { + offset += size; + } + + public void removeNonparsedBuffer() + { + used = offset; + } + + /* + public static class Context { + private boolean finished; + private Object data; + private int offset; + private UnpackerImpl impl; + + public Context() + { + this.finished = false; + this.impl = new UnpackerImpl(); + } + + public boolean isFinished() + { + return finished; + } + + public Object getData() + { + return data; + } + + int getOffset() + { + return offset; + } + + void setFinished(boolean finished) + { + this.finished = finished; + } + + void setData(Object data) + { + this.data = data; + } + + void setOffset(int offset) + { + this.offset = offset; + } + + UnpackerImpl getImpl() + { + return impl; + } + } + + public static int unpack(Context ctx, byte[] buffer) throws UnpackException + { + return unpack(ctx, buffer, 0, buffer.length); + } + + public static int unpack(Context ctx, byte[] buffer, int offset, int length) throws UnpackException + { + UnpackerImpl impl = ctx.getImpl(); + int noffset = impl.execute(buffer, offset + ctx.getOffset(), length); + ctx.setOffset(noffset - offset); + if(impl.isFinished()) { + ctx.setData(impl.getData()); + ctx.setFinished(false); + impl.reset(); + } else { + ctx.setData(null); + ctx.setFinished(true); + } + int parsed = noffset - offset; + ctx.setOffset(parsed); + return noffset; + } + */ +} + diff --git a/java-plan2/src/org/msgpack/impl/ArrayBuilder.java b/java-plan2/src/org/msgpack/impl/ArrayBuilder.java new file mode 100644 index 0000000..9bb099b --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/ArrayBuilder.java @@ -0,0 +1,7 @@ +package org.msgpack.impl; + +public interface ArrayBuilder { + public void add(Object element); + public Object finish(); +} + diff --git a/java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java b/java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java new file mode 100644 index 0000000..814e302 --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java @@ -0,0 +1,127 @@ +package org.msgpack.impl; + +import org.msgpack.*; + +public class GenericObjectBuilder implements ObjectBuilder { + public Object createNil() + { + return null; + } + + @Override + public Object createBoolean(boolean v) + { + return new GenericBoolean(v); + } + + @Override + public Object createByte(byte v) + { + //return new GenericByte(v); + return null; // FIXME + } + + @Override + public Object createShort(short v) + { + return new GenericShort(v); + } + + @Override + public Object createInt(int v) + { + //return new GenericInt(v); + return null; // FIXME + } + + @Override + public Object createLong(long v) + { + return new GenericLong(v); + } + + @Override + public Object createFloat(float v) + { + //return new GenericFloat(v); + return null; // FIXME + } + + @Override + public Object createDouble(double v) + { + //return new GenericDouble(v); + return null; // FIXME + } + + @Override + public Object createRaw(byte[] b, int offset, int length) + { + byte[] copy = new byte[length]; + System.arraycopy(b, offset, copy, 0, length); + return new GenericRaw(copy); + } + + @Override + public ArrayBuilder createArray(int length) + { + return new GenericArrayBuilder(length); + } + + @Override + public MapBuilder createMap(int length) + { + return new GenericMapBuilder(length); + } +} + +final class GenericArrayBuilder implements ArrayBuilder { + private GenericArray a; + + GenericArrayBuilder(int length) + { + this.a = new GenericArray(length); + } + + @Override + public void add(Object element) + { + a.add((GenericObject)element); + } + + @Override + public Object finish() + { + return a; + } +} + +final class GenericMapBuilder implements MapBuilder { + private GenericMap m; + private GenericObject key; + + GenericMapBuilder(int length) + { + this.m = new GenericMap(length); + } + + @Override + public void putKey(Object key) + { + this.key = (GenericObject)key; + } + + @Override + public void putValue(Object value) + { + m.put(this.key, (GenericObject)value); + this.key = null; + } + + @Override + public Object finish() + { + return m; + } +} + diff --git a/java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java b/java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java new file mode 100644 index 0000000..5569fbf --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java @@ -0,0 +1,12 @@ +package org.msgpack.impl; + +import org.msgpack.*; + +public class GenericZeroCopyObjectBuilder extends GenericObjectBuilder { + @Override + public Object createRaw(byte[] b, int offset, int length) + { + return new GenericRawRef(b, offset, length); + } +} + diff --git a/java-plan2/src/org/msgpack/impl/MapBuilder.java b/java-plan2/src/org/msgpack/impl/MapBuilder.java new file mode 100644 index 0000000..57859a6 --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/MapBuilder.java @@ -0,0 +1,8 @@ +package org.msgpack.impl; + +public interface MapBuilder { + public void putKey(Object key); + public void putValue(Object value); + public Object finish(); +} + diff --git a/java-plan2/src/org/msgpack/impl/ObjectBuilder.java b/java-plan2/src/org/msgpack/impl/ObjectBuilder.java new file mode 100644 index 0000000..3268903 --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/ObjectBuilder.java @@ -0,0 +1,16 @@ +package org.msgpack.impl; + +public interface ObjectBuilder { + public Object createNil(); + public Object createBoolean(boolean v); + public Object createByte(byte v); + public Object createShort(short v); + public Object createInt(int v); + public Object createLong(long v); + public Object createFloat(float v); + public Object createDouble(double v); + public Object createRaw(byte[] b, int offset, int length); + public ArrayBuilder createArray(int length); + public MapBuilder createMap(int length); +} + diff --git a/java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java b/java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java new file mode 100644 index 0000000..7748844 --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java @@ -0,0 +1,203 @@ +package org.msgpack.impl; + +import java.util.List; +import java.util.ArrayList; +import java.util.HashMap; +import org.msgpack.schema.*; + +public final class SpecificObjectBuilder implements ObjectBuilder { + private int top; + private Schema schema; + + public SpecificObjectBuilder(Schema schema) + { + this.top = 0; + this.schema = schema; + } + + void setSchema(Schema s) + { + schema = s; + } + + Schema swapSchema(Schema s) + { + Schema old = schema; + schema = s; + return old; + } + + @Override + public Object createNil() + { + return schema.createNil(); + } + + @Override + public Object createBoolean(boolean v) + { + return schema.createBoolean(v); + } + + @Override + public Object createByte(byte v) + { + return schema.createByte(v); + } + + @Override + public Object createShort(short v) + { + return schema.createShort(v); + } + + @Override + public Object createInt(int v) + { + return schema.createInt(v); + } + + @Override + public Object createLong(long v) + { + return schema.createLong(v); + } + + @Override + public Object createFloat(float v) + { + return schema.createFloat(v); + } + + @Override + public Object createDouble(double v) + { + return schema.createDouble(v); + } + + @Override + public Object createRaw(byte[] b, int offset, int length) + { + return schema.createRaw(b, offset, length); + } + + @Override + public ArrayBuilder createArray(int length) + { + if(schema instanceof ClassSchema) { + return new ClassBuilder(length, (ClassSchema)schema, this); + } + ArraySchema as = (ArraySchema)schema; + return new SpecificArrayBuilder(length, as.getElementType(), this); + } + + @Override + public MapBuilder createMap(int length) + { + MapSchema ms = (MapSchema)schema; + return new SpecificMapBuilder(length, ms.getKeyType(), ms.getValueType(), this); + } +} + +final class SpecificArrayBuilder implements ArrayBuilder { + private ArrayList a; + private SpecificObjectBuilder builder; + private Schema parentSchema; + + SpecificArrayBuilder(int length, Schema elementSchema, SpecificObjectBuilder builder) + { + this.a = new ArrayList(length); + this.builder = builder; + this.parentSchema = builder.swapSchema(elementSchema); + } + + public void add(Object element) + { + a.add(element); + } + + public Object finish() + { + builder.swapSchema(parentSchema); + return a; + } +} + +final class SpecificMapBuilder implements MapBuilder { + private HashMap m; + private Object key; + private Schema keySchema; + private Schema valueSchema; + private SpecificObjectBuilder builder; + private Schema parentSchema; + + SpecificMapBuilder(int length, Schema keySchema, Schema valueSchema, SpecificObjectBuilder builder) + { + this.m = new HashMap(length); + this.keySchema = keySchema; + this.valueSchema = valueSchema; + this.builder = builder; + this.parentSchema = builder.swapSchema(keySchema); + } + + @Override + public void putKey(Object key) + { + this.key = key; + this.builder.setSchema(valueSchema); + } + + @Override + public void putValue(Object value) + { + m.put(this.key, value); + this.key = null; + this.builder.setSchema(keySchema); + } + + @Override + public Object finish() + { + builder.swapSchema(parentSchema); + return m; + } +} + +final class ClassBuilder implements ArrayBuilder { + private Object object; + private int index; + private List fields; + private SpecificObjectBuilder builder; + private Schema parentSchema; + + ClassBuilder(int length, ClassSchema schema, SpecificObjectBuilder builder) + { + this.object = schema.newInstance(); + this.index = 0; + this.fields = schema.getFields(); + this.builder = builder; + this.parentSchema = builder.swapSchema(fields.get(0).getType()); + // FIXME check length + } + + @Override + public void add(Object element) + { + FieldSchema f = fields.get(index++); // FIXME check fields.size + f.setFieldValue(object, element); // XXX FIXME debug + if(fields.size() > index) { + builder.setSchema( fields.get(index).getType() ); + } else { + builder.setSchema( null ); + // FIXME: builder.setSchema(new InvalidFieldSchema); + } + } + + @Override + public Object finish() + { + builder.swapSchema(parentSchema); + return object; + } +} + diff --git a/java-plan2/src/org/msgpack/impl/UnpackerImpl.java b/java-plan2/src/org/msgpack/impl/UnpackerImpl.java new file mode 100644 index 0000000..9f88072 --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/UnpackerImpl.java @@ -0,0 +1,395 @@ +package org.msgpack.impl; + +import java.nio.ByteBuffer; +//import java.math.BigInteger; +import org.msgpack.UnpackException; + +public class UnpackerImpl { + static final int CS_HEADER = 0x00; + static final int CS_FLOAT = 0x0a; + static final int CS_DOUBLE = 0x0b; + static final int CS_UINT_8 = 0x0c; + static final int CS_UINT_16 = 0x0d; + static final int CS_UINT_32 = 0x0e; + static final int CS_UINT_64 = 0x0f; + static final int CS_INT_8 = 0x10; + static final int CS_INT_16 = 0x11; + static final int CS_INT_32 = 0x12; + static final int CS_INT_64 = 0x13; + static final int CS_RAW_16 = 0x1a; + static final int CS_RAW_32 = 0x1b; + static final int CS_ARRAY_16 = 0x1c; + static final int CS_ARRAY_32 = 0x1d; + static final int CS_MAP_16 = 0x1e; + static final int CS_MAP_32 = 0x1f; + static final int ACS_RAW_VALUE = 0x20; + static final int CT_ARRAY_ITEM = 0x00; + static final int CT_MAP_KEY = 0x01; + static final int CT_MAP_VALUE = 0x02; + + static final int MAX_STACK_SIZE = 16; + + protected int cs = CS_HEADER; + protected int trail = 0; + protected int top = -1; + protected boolean finished = false; + protected Object data = null; + protected int[] stack_ct = new int[MAX_STACK_SIZE]; + protected int[] stack_count = new int[MAX_STACK_SIZE]; + protected Object[] stack_obj = new Object[MAX_STACK_SIZE]; + protected ByteBuffer castBuffer = ByteBuffer.allocate(8); + protected ObjectBuilder builder; + + protected UnpackerImpl(ObjectBuilder builder) + { + this.builder = builder; + } + + protected void setBuilder(ObjectBuilder builder) + { + this.builder = builder; + } + + protected Object getData() + { + return data; + } + + protected boolean isFinished() + { + return finished; + } + + protected void reset() + { + for(int i=0; i <= top; ++top) { + stack_ct[top] = 0; + stack_count[top] = 0; + stack_obj[top] = null; + } + cs = CS_HEADER; + trail = 0; + top = -1; + finished = false; + data = null; + } + + @SuppressWarnings("unchecked") + protected int execute(byte[] src, int off, int length) throws UnpackException + { + if(off >= length) { return off; } + + int limit = length; + int i = off; + int count; + + Object obj = null; + + _out: do { + _header_again: { + //System.out.println("while i:"+i+" limit:"+limit); + + int b = src[i]; + + _push: { + _fixed_trail_again: + if(cs == CS_HEADER) { + + if((b & 0x80) == 0) { // Positive Fixnum + //System.out.println("positive fixnum "+b); + obj = builder.createByte((byte)b); + break _push; + } + + if((b & 0xe0) == 0xe0) { // Negative Fixnum + //System.out.println("negative fixnum "+b); + obj = builder.createByte((byte)b); + break _push; + } + + if((b & 0xe0) == 0xa0) { // FixRaw + trail = b & 0x1f; + if(trail == 0) { + obj = builder.createRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + } + + if((b & 0xf0) == 0x90) { // FixArray + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + count = b & 0x0f; + ++top; + stack_obj[top] = builder.createArray(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + //System.out.println("fixarray count:"+count); + break _header_again; + } + + if((b & 0xf0) == 0x80) { // FixMap + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + count = b & 0x0f; + ++top; + stack_obj[top] = builder.createMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + } + + switch(b & 0xff) { // FIXME + case 0xc0: // nil + obj = builder.createNil(); + break _push; + case 0xc2: // false + obj = builder.createBoolean(false); + break _push; + case 0xc3: // true + obj = builder.createBoolean(true); + break _push; + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + trail = 1 << (b & 0x03); + cs = b & 0x1f; + //System.out.println("a trail "+trail+" cs:"+cs); + break _fixed_trail_again; + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + trail = 2 << (b & 0x01); + cs = b & 0x1f; + //System.out.println("b trail "+trail+" cs:"+cs); + break _fixed_trail_again; + default: + //System.out.println("unknown b "+(b&0xff)); + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + + } // _fixed_trail_again + + do { + _fixed_trail_again: { + + if(limit - i <= trail) { break _out; } + int n = i + 1; + i += trail; + + switch(cs) { + case CS_FLOAT: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = builder.createFloat( castBuffer.getFloat(0) ); + //System.out.println("float "+obj); + break _push; + case CS_DOUBLE: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + obj = builder.createDouble( castBuffer.getDouble(0) ); + //System.out.println("double "+obj); + break _push; + case CS_UINT_8: + //System.out.println(n); + //System.out.println(src[n]); + //System.out.println(src[n+1]); + //System.out.println(src[n-1]); + obj = builder.createShort( (short)((src[n]) & 0xff) ); + //System.out.println("uint8 "+obj); + break _push; + case CS_UINT_16: + //System.out.println(src[n]); + //System.out.println(src[n+1]); + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = builder.createInt( ((int)castBuffer.getShort(0)) & 0xffff ); + //System.out.println("uint 16 "+obj); + break _push; + case CS_UINT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = builder.createLong( ((long)castBuffer.getInt(0)) & 0xffffffffL ); + //System.out.println("uint 32 "+obj); + break _push; + case CS_UINT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + { + long o = castBuffer.getLong(0); + if(o < 0) { + // FIXME + //obj = GenericBigInteger.valueOf(o & 0x7fffffffL).setBit(31); + } else { + obj = builder.createLong( o ); + } + } + throw new UnpackException("uint 64 bigger than 0x7fffffff is not supported", UnpackException.PARSE_ERROR); + case CS_INT_8: + obj = builder.createByte( src[n] ); + break _push; + case CS_INT_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = builder.createShort( castBuffer.getShort(0) ); + break _push; + case CS_INT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = builder.createInt( castBuffer.getInt(0) ); + break _push; + case CS_INT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + obj = builder.createLong( castBuffer.getLong(0) ); + break _push; + case CS_RAW_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + trail = ((int)castBuffer.getShort(0)) & 0xffff; + if(trail == 0) { + obj = builder.createRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + case CS_RAW_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + trail = castBuffer.getInt(0) & 0x7fffffff; + if(trail == 0) { + obj = builder.createRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + case ACS_RAW_VALUE: + obj = builder.createRaw(src, n, trail); + break _push; + case CS_ARRAY_16: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + ++top; + stack_obj[top] = builder.createArray(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + break _header_again; + case CS_ARRAY_32: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + ++top; + stack_obj[top] = builder.createArray(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + break _header_again; + case CS_MAP_16: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + ++top; + stack_obj[top] = builder.createMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + case CS_MAP_32: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + ++top; + stack_obj[top] = builder.createMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + default: + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + + } // _fixed_trail_again + } while(true); + } // _push + + do { + _push: { + //System.out.println("push top:"+top); + if(top == -1) { + ++i; + data = obj; + finished = true; + break _out; + } + + switch(stack_ct[top]) { + case CT_ARRAY_ITEM: + //System.out.println("array item "+obj); + ((ArrayBuilder)stack_obj[top]).add(obj); + if(--stack_count[top] == 0) { + obj = ((ArrayBuilder)stack_obj[top]).finish(); + stack_obj[top] = null; + --top; + break _push; + } + break _header_again; + case CT_MAP_KEY: + //System.out.println("map key:"+top+" "+obj); + MapBuilder mb = (MapBuilder)stack_obj[top]; + mb.putKey(obj); + stack_ct[top] = CT_MAP_VALUE; + break _header_again; + case CT_MAP_VALUE: + //System.out.println("map value:"+top+" "+obj); + ((MapBuilder)stack_obj[top]).putValue(obj); + if(--stack_count[top] == 0) { + obj = ((MapBuilder)stack_obj[top]).finish(); + stack_obj[top] = null; + --top; + break _push; + } + stack_ct[top] = CT_MAP_KEY; + break _header_again; + default: + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + } // _push + } while(true); + + } // _header_again + cs = CS_HEADER; + ++i; + } while(i < limit); // _out + + return i; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/ArraySchema.java b/java-plan2/src/org/msgpack/schema/ArraySchema.java new file mode 100644 index 0000000..4b05190 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/ArraySchema.java @@ -0,0 +1,61 @@ +package org.msgpack.schema; + +import java.util.List; +import java.util.ArrayList; +import java.io.IOException; +import org.msgpack.*; + +public class ArraySchema extends Schema { + private Schema elementType; + + public ArraySchema(Schema elementType) + { + super("array"); + this.elementType = elementType; + } + + public Schema getElementType() + { + return elementType; + } + + @Override + public String getFullName() + { + return "ArrayList<"+elementType.getFullName()+">"; + } + + @Override + public String getExpression() + { + return "(array "+elementType.getExpression()+")"; + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + List d = (List)obj; + pk.packArray(d.size()); + for(Object e : d) { + elementType.pack(pk, e); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(GenericObject obj) + { + List d = obj.asArray(); + List g = new ArrayList(); + for(GenericObject o : d) { + g.add( elementType.convert(o) ); + } + return g; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/ClassGenerator.java b/java-plan2/src/org/msgpack/schema/ClassGenerator.java new file mode 100644 index 0000000..25a9620 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/ClassGenerator.java @@ -0,0 +1,230 @@ +package org.msgpack.schema; + +import java.util.ArrayList; +import java.util.List; +import java.io.IOException; +import java.io.File; +import java.io.FileOutputStream; +import java.io.Writer; + +public class ClassGenerator { + private ClassSchema schema; + private Writer writer; + private int indent; + + private ClassGenerator(Writer writer) + { + this.writer = writer; + this.indent = 0; + } + + public static void write(Schema schema, Writer dest) throws IOException + { + if(!(schema instanceof ClassSchema)) { + throw new RuntimeException("schema is not class schema"); + } + ClassSchema cs = (ClassSchema)schema; + new ClassGenerator(dest).run(cs); + } + + private void run(ClassSchema cs) throws IOException + { + List subclasses = new ArrayList(); + for(FieldSchema f : cs.getFields()) { + findSubclassSchema(subclasses, f.getType()); + } + + for(ClassSchema sub : subclasses) { + sub.setNamespace(cs.getNamespace()); + sub.setImports(cs.getImports()); + } + + this.schema = cs; + + writeHeader(); + + writeClass(); + + for(ClassSchema sub : subclasses) { + this.schema = sub; + writeSubclass(); + } + + writeFooter(); + + this.schema = null; + writer.flush(); + } + + private void findSubclassSchema(List dst, Schema s) + { + if(s instanceof ClassSchema) { + ClassSchema cs = (ClassSchema)s; + if(!dst.contains(cs)) { dst.add(cs); } + for(FieldSchema f : cs.getFields()) { + findSubclassSchema(dst, f.getType()); + } + } else if(s instanceof ArraySchema) { + ArraySchema as = (ArraySchema)s; + findSubclassSchema(dst, as.getElementType()); + } else if(s instanceof MapSchema) { + MapSchema as = (MapSchema)s; + findSubclassSchema(dst, as.getKeyType()); + findSubclassSchema(dst, as.getValueType()); + } + } + + private void writeHeader() throws IOException + { + if(schema.getNamespace() != null) { + line("package "+schema.getNamespace()+";"); + line(); + } + line("import java.util.*;"); + line("import java.io.*;"); + line("import org.msgpack.*;"); + line("import org.msgpack.schema.*;"); + } + + private void writeFooter() throws IOException + { + } + + private void writeClass() throws IOException + { + line(); + line("public final class "+schema.getName()+" implements MessagePackable, MessageConvertable"); + line("{"); + pushIndent(); + writeSchema(); + writeMemberVariables(); + writeMemberFunctions(); + popIndent(); + line("}"); + } + + private void writeSubclass() throws IOException + { + line(); + line("final class "+schema.getName()+" implements MessagePackable, MessageConvertable"); + line("{"); + pushIndent(); + writeSchema(); + writeMemberVariables(); + writeMemberFunctions(); + popIndent(); + line("}"); + } + + private void writeSchema() throws IOException + { + line("private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load(\""+schema.getExpression()+"\");"); + line("public static ClassSchema getSchema() { return _SCHEMA; }"); + } + + private void writeMemberVariables() throws IOException + { + line(); + for(FieldSchema f : schema.getFields()) { + line("public "+f.getType().getFullName()+" "+f.getName()+";"); + } + } + + private void writeMemberFunctions() throws IOException + { + // void messagePack(Packer pk) + // boolean equals(Object obj) + // int hashCode() + // void set(int _index, Object _value) + // Object get(int _index); + // getXxx() + // setXxx(Xxx xxx) + writeConstructors(); + writeAccessors(); + writePackFunction(); + writeConvertFunction(); + } + + private void writeConstructors() throws IOException + { + line(); + line("public "+schema.getName()+"() { }"); + } + + private void writeAccessors() throws IOException + { + // FIXME + //line(); + //for(FieldSchema f : schema.getFields()) { + // line(""); + //} + } + + private void writePackFunction() throws IOException + { + line(); + line("@Override"); + line("public void messagePack(Packer pk) throws IOException"); + line("{"); + pushIndent(); + line("List _f = _SCHEMA.getFields();"); + line("pk.packArray("+schema.getFields().size()+");"); + int i = 0; + for(FieldSchema f : schema.getFields()) { + line("_f.get("+i+").getType().pack(pk, "+f.getName()+");"); + ++i; + } + popIndent(); + line("}"); + } + + private void writeConvertFunction() throws IOException + { + line(); + line("@Override"); + line("@SuppressWarnings(\"unchecked\")"); + line("public void messageConvert(GenericObject obj)"); + line("{"); + pushIndent(); + line("List _l = obj.asArray();"); + line("List _f = _SCHEMA.getFields();"); + int i = 0; + for(FieldSchema f : schema.getFields()) { + line("if(_l.size() <= "+i+") { return; } "+f.getName()+" = ("+f.getType().getFullName()+")_f.get("+i+").getType().convert(_l.get("+i+"));"); + ++i; + } + popIndent(); + line("}"); + line(); + line("public static "+schema.getName()+" convert(GenericObject obj)"); + line("{"); + pushIndent(); + line("return ("+schema.getName()+")_SCHEMA.convert(obj);"); + popIndent(); + line("}"); + } + + private void line(String str) throws IOException + { + for(int i=0; i < indent; ++i) { + writer.write("\t"); + } + writer.write(str+"\n"); + } + + private void line() throws IOException + { + writer.write("\n"); + } + + private void pushIndent() + { + indent += 1; + } + + private void popIndent() + { + indent -= 1; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/ClassSchema.java b/java-plan2/src/org/msgpack/schema/ClassSchema.java new file mode 100644 index 0000000..3343fca --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/ClassSchema.java @@ -0,0 +1,39 @@ +package org.msgpack.schema; + +import java.util.List; + +public abstract class ClassSchema extends Schema { + protected String namespace; + protected List imports; + + public ClassSchema(String name, String namespace, List imports) + { + super(name); + this.namespace = namespace; + this.imports = imports; + } + + public String getNamespace() + { + return namespace; + } + + public List getImports() + { + return imports; + } + + void setNamespace(String namespace) + { + this.namespace = namespace; + } + + void setImports(List imports) + { + this.imports = imports; + } + + public abstract List getFields(); + public abstract Object newInstance(); +} + diff --git a/java-plan2/src/org/msgpack/schema/FieldSchema.java b/java-plan2/src/org/msgpack/schema/FieldSchema.java new file mode 100644 index 0000000..31a132c --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/FieldSchema.java @@ -0,0 +1,31 @@ +package org.msgpack.schema; + +public abstract class FieldSchema { + private String name; + private Schema type; + + public FieldSchema(String name, Schema type) + { + this.name = name; + this.type = type; + } + + public String getName() + { + return this.name; + } + + public Schema getType() + { + return type; + } + + public String getExpression() + { + return "(field "+name+" "+type.getExpression()+")"; + } + + public abstract Object getFieldValue(Object obj); + public abstract void setFieldValue(Object obj, Object value); +} + diff --git a/java-plan2/src/org/msgpack/schema/GenericClassSchema.java b/java-plan2/src/org/msgpack/schema/GenericClassSchema.java new file mode 100644 index 0000000..f1e2b44 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/GenericClassSchema.java @@ -0,0 +1,89 @@ +package org.msgpack.schema; + +import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import org.msgpack.*; + + +public class GenericClassSchema extends ClassSchema { + private List fields; + + private String fqdn; + private Constructor constructorCache; + + public GenericClassSchema(String name, List fields, String namespace, List imports) + { + super(name, namespace, imports); + this.fields = fields; + if(namespace == null) { + this.fqdn = name; + } else { + this.fqdn = namespace+"."+name; + } + } + + //@Override + //public String getFullName() + //{ + // if(namespace == null) { + // return getName(); + // } else { + // return namespace+"."+getName(); + // } + //} + + public List getFields() + { + return fields; + } + + public String getNamespace() + { + return namespace; + } + + @Override + public String getExpression() + { + StringBuffer b = new StringBuffer(); + b.append("(class "); + b.append(getName()); + if(namespace != null) { + b.append(" (package "+namespace+")"); + } + for(GenericFieldSchema f : fields) { + b.append(" "+f.getExpression()); + } + b.append(")"); + return b.toString(); + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + // FIXME + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(GenericObject obj) + { + // FIXME + return obj; + } + + @Override + public Object newInstance() + { + return new HashMap(fields.size()); + } +} + diff --git a/java-plan2/src/org/msgpack/schema/GenericFieldSchema.java b/java-plan2/src/org/msgpack/schema/GenericFieldSchema.java new file mode 100644 index 0000000..507ee18 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/GenericFieldSchema.java @@ -0,0 +1,25 @@ +package org.msgpack.schema; + +import java.util.Map; +import java.lang.reflect.Field; + +public final class GenericFieldSchema extends FieldSchema { + public GenericFieldSchema(String name, Schema type) + { + super(name, type); + } + + @Override + public Object getFieldValue(Object obj) + { + return ((Map)obj).get(getName()); + } + + @Override + @SuppressWarnings("unchecked") + public void setFieldValue(Object obj, Object value) + { + ((Map)obj).put(getName(), value); + } +} + diff --git a/java-plan2/src/org/msgpack/schema/IntSchema.java b/java-plan2/src/org/msgpack/schema/IntSchema.java new file mode 100644 index 0000000..69771c3 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/IntSchema.java @@ -0,0 +1,58 @@ +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class IntSchema extends Schema { + public IntSchema() + { + super("Integer"); + } + + @Override + public String getExpression() + { + return "int"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + pk.packInt((Integer)obj); + } + + @Override + public Object convert(GenericObject obj) + { + return obj.asInt(); + } + + @Override + public Object createByte(byte v) + { + return (int)v; + } + + @Override + public Object createShort(short v) + { + return (int)v; + } + + @Override + public Object createInt(int v) + { + return (int)v; + } + + @Override + public Object createLong(long v) + { + return (int)v; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/LongSchema.java b/java-plan2/src/org/msgpack/schema/LongSchema.java new file mode 100644 index 0000000..0ba3057 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/LongSchema.java @@ -0,0 +1,58 @@ +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class LongSchema extends Schema { + public LongSchema() + { + super("Long"); + } + + @Override + public String getExpression() + { + return "long"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + pk.packLong((Long)obj); + } + + @Override + public Object convert(GenericObject obj) + { + return obj.asLong(); + } + + @Override + public Object createByte(byte v) + { + return (long)v; + } + + @Override + public Object createShort(short v) + { + return (long)v; + } + + @Override + public Object createInt(int v) + { + return (long)v; + } + + @Override + public Object createLong(long v) + { + return (long)v; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/MapSchema.java b/java-plan2/src/org/msgpack/schema/MapSchema.java new file mode 100644 index 0000000..e72cf63 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/MapSchema.java @@ -0,0 +1,70 @@ +package org.msgpack.schema; + +import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.io.IOException; +import org.msgpack.*; + +public class MapSchema extends Schema { + private Schema keyType; + private Schema valueType; + + public MapSchema(Schema keyType, Schema valueType) + { + super("map"); + this.keyType = keyType; + this.valueType = valueType; + } + + public Schema getKeyType() + { + return keyType; + } + + public Schema getValueType() + { + return valueType; + } + + @Override + public String getFullName() + { + return "HashList<"+keyType.getFullName()+", "+valueType.getFullName()+">"; + } + + @Override + public String getExpression() + { + return "(map "+keyType.getExpression()+" "+valueType.getExpression()+")"; + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + Map d = (Map)obj; + pk.packMap(d.size()); + for(Map.Entry e : d.entrySet()) { + keyType.pack(pk, e.getKey()); + valueType.pack(pk, e.getValue()); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(GenericObject obj) + { + Map d = obj.asMap(); + Map g = new HashMap(); + for(Map.Entry e : d.entrySet()) { + g.put(keyType.convert(e.getKey()), valueType.convert(e.getValue())); + } + return g; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/ObjectSchema.java b/java-plan2/src/org/msgpack/schema/ObjectSchema.java new file mode 100644 index 0000000..a9f30f5 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/ObjectSchema.java @@ -0,0 +1,33 @@ +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class ObjectSchema extends Schema { + public ObjectSchema() + { + super("object"); + } + + public String getFullName() + { + return "GenericObject"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + pk.pack(obj); + } + + @Override + public Object convert(GenericObject obj) + { + return obj; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/PrimitiveSchema.java b/java-plan2/src/org/msgpack/schema/PrimitiveSchema.java new file mode 100644 index 0000000..023d81b --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/PrimitiveSchema.java @@ -0,0 +1,21 @@ +package org.msgpack.schema; + +public abstract class PrimitiveSchema extends Schema { + public static enum PrimitiveType { + BYTE, + SHORT, + INT, + LONG, + FLOAT, + DOUBLE, + } + + public final PrimitiveType type; + + protected PrimitiveSchema(String name, PrimitiveType type) + { + super(name); + this.type = type; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/RawSchema.java b/java-plan2/src/org/msgpack/schema/RawSchema.java new file mode 100644 index 0000000..847ad29 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/RawSchema.java @@ -0,0 +1,44 @@ +package org.msgpack.schema; + +import java.io.IOException; +import java.nio.charset.Charset; +import org.msgpack.*; + +public class RawSchema extends Schema { + public RawSchema() + { + super("raw"); + } + + public String getFullName() + { + return "byte[]"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + byte[] d = (byte[])obj; + pk.packRaw(d.length); + pk.packRawBody(d); + } + + @Override + public Object convert(GenericObject obj) + { + return obj.asBytes(); + } + + @Override + public Object createRaw(byte[] b, int offset, int length) + { + byte[] d = new byte[length]; + System.arraycopy(b, offset, d, 0, length); + return d; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/SSchemaParser.java b/java-plan2/src/org/msgpack/schema/SSchemaParser.java new file mode 100644 index 0000000..bfe912f --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/SSchemaParser.java @@ -0,0 +1,246 @@ +package org.msgpack.schema; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Stack; +import java.util.regex.Pattern; +import java.util.regex.Matcher; + +// FIXME exception class + +class SSchemaParser { + public static Schema parse(String source) + { + return new SSchemaParser(false).run(source); + } + + public static Schema load(String source) + { + return new SSchemaParser(true).run(source); + } + + private static abstract class SExp { + boolean isAtom() { return false; } + public String getAtom() { return null; } + + boolean isTuple() { return false; } + public SExp getTuple(int i) { return null; } + public int size() { return 0; } + public boolean empty() { return size() == 0; } + Iterator iterator(int offset) { return null; } + } + + private static class SAtom extends SExp { + private String atom; + + SAtom(String atom) { this.atom = atom; } + + boolean isAtom() { return true; } + public String getAtom() { return atom; } + + public String toString() { return atom; } + } + + private static class STuple extends SExp { + private List tuple; + + STuple() { this.tuple = new ArrayList(); } + + public void add(SExp e) { tuple.add(e); } + + boolean isTuple() { return true; } + public SExp getTuple(int i) { return tuple.get(i); } + public int size() { return tuple.size(); } + + Iterator iterator(int skip) { + Iterator i = tuple.iterator(); + for(int s=0; s < skip; ++s) { i.next(); } + return i; + } + + public String toString() { + if(tuple.isEmpty()) { return "()"; } + Iterator i = tuple.iterator(); + StringBuffer o = new StringBuffer(); + o.append("(").append(i.next()); + while(i.hasNext()) { o.append(" ").append(i.next()); } + o.append(")"); + return o.toString(); + } + } + + boolean specificClass; + + private SSchemaParser(boolean specificClass) + { + this.specificClass = specificClass; + } + + private static Pattern pattern = Pattern.compile( + "(?:\\s+)|([\\(\\)]|[\\d\\w\\.]+)"); + + private Schema run(String source) + { + Matcher m = pattern.matcher(source); + + Stack stack = new Stack(); + String token; + + while(true) { + while(true) { + if(!m.find()) { throw new RuntimeException("unexpected end of file"); } + token = m.group(1); + if(token != null) { break; } + } + + if(token.equals("(")) { + stack.push(new STuple()); + } else if(token.equals(")")) { + STuple top = stack.pop(); + if(stack.empty()) { + stack.push(top); + break; + } + stack.peek().add(top); + } else { + if(stack.empty()) { + throw new RuntimeException("unexpected token '"+token+"'"); + } + stack.peek().add(new SAtom(token)); + } + } + + while(true) { + if(!m.find()) { break; } + token = m.group(1); + if(token != null) { throw new RuntimeException("unexpected token '"+token+"'"); } + } + + return readType( stack.pop() ); + } + + private Schema readType(SExp exp) + { + if(exp.isAtom()) { + String type = exp.getAtom(); + // FIXME + if(type.equals("string")) { + return new StringSchema(); + } else if(type.equals("raw")) { + return new RawSchema(); + } else if(type.equals("short")) { + return new ShortSchema(); + } else if(type.equals("int")) { + return new IntSchema(); + } else if(type.equals("long")) { + return new LongSchema(); + } else if(type.equals("object")) { + return new ObjectSchema(); + } else { + throw new RuntimeException("byte, short, int, long, float, double, raw, string or object is expected but got '"+type+"': "+exp); + } + } else { + String type = exp.getTuple(0).getAtom(); + if(type.equals("class")) { + return parseClass(exp); + } else if(type.equals("array")) { + return parseArray(exp); + } else if(type.equals("map")) { + return parseMap(exp); + } else { + throw new RuntimeException("class, array or map is expected but got '"+type+"': "+exp); + } + } + } + + private ClassSchema parseClass(SExp exp) + { + if(exp.size() < 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("class is (class NAME CLASS_BODY): "+exp); + } + + String namespace = null; + List imports = new ArrayList(); + String name = exp.getTuple(1).getAtom(); + List fields = new ArrayList(); + + for(Iterator i=exp.iterator(2); i.hasNext();) { + SExp subexp = i.next(); + if(!subexp.isTuple() || subexp.empty() || !subexp.getTuple(0).isAtom()) { + throw new RuntimeException("field, package or import is expected: "+subexp); + } + String type = subexp.getTuple(0).getAtom(); + if(type.equals("field")) { + fields.add( parseField(subexp) ); + } else if(type.equals("package")) { + if(namespace != null) { + throw new RuntimeException("duplicated package definition: "+subexp); + } + namespace = parseNamespace(subexp); + } else if(type.equals("import")) { + imports.add( parseImport(subexp) ); + } else { + throw new RuntimeException("field, package or import is expected but got '"+type+"': "+subexp); + } + } + + if(specificClass) { + return new SpecificClassSchema(name, fields, namespace, imports); + } else { + return new GenericClassSchema(name, fields, namespace, imports); + } + } + + private ArraySchema parseArray(SExp exp) + { + if(exp.size() != 2) { + throw new RuntimeException("array is (array ELEMENT_TYPE): "+exp); + } + Schema elementType = readType(exp.getTuple(1)); + return new ArraySchema(elementType); + } + + private MapSchema parseMap(SExp exp) + { + if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("map is (map KEY_TYPE VALUE_TYPE): "+exp); + } + Schema keyType = readType(exp.getTuple(1)); + Schema valueType = readType(exp.getTuple(2)); + return new MapSchema(keyType, valueType); + } + + private String parseNamespace(SExp exp) + { + if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("package is (package NAME): "+exp); + } + String name = exp.getTuple(1).getAtom(); + return name; + } + + private String parseImport(SExp exp) + { + if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("import is (import NAME): "+exp); + } + String name = exp.getTuple(1).getAtom(); + return name; + } + + private FieldSchema parseField(SExp exp) + { + if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("field is (field NAME TYPE): "+exp); + } + String name = exp.getTuple(1).getAtom(); + Schema type = readType(exp.getTuple(2)); + if(specificClass) { + return new SpecificFieldSchema(name, type); + } else { + return new GenericFieldSchema(name, type); + } + } +} + diff --git a/java-plan2/src/org/msgpack/schema/Schema.java b/java-plan2/src/org/msgpack/schema/Schema.java new file mode 100644 index 0000000..15b7e72 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/Schema.java @@ -0,0 +1,93 @@ +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.impl.*; +import org.msgpack.Packer; +import org.msgpack.GenericObject; + +public abstract class Schema { + private String expression; + private String name; + + public Schema(String name) + { + this.expression = expression; + this.name = name; + } + + public String getName() + { + return name; + } + + public String getFullName() + { + return name; + } + + public String getExpression() + { + return name; + } + + public static Schema parse(String source) + { + return SSchemaParser.parse(source); + } + + public static Schema load(String source) + { + return SSchemaParser.load(source); + } + + public abstract void pack(Packer pk, Object obj) throws IOException; + public abstract Object convert(GenericObject obj); + //public abstract Object convertGeneric(GenericObject obj); + + + public Object createNil() + { + return null; + } + + public Object createBoolean(boolean v) + { + throw new RuntimeException("type error"); + } + + public Object createByte(byte v) + { + throw new RuntimeException("type error"); + } + + public Object createShort(short v) + { + throw new RuntimeException("type error"); + } + + public Object createInt(int v) + { + throw new RuntimeException("type error"); + } + + public Object createLong(long v) + { + throw new RuntimeException("type error"); + } + + public Object createFloat(float v) + { + throw new RuntimeException("type error"); + } + + public Object createDouble(double v) + { + throw new RuntimeException("type error"); + } + + public Object createRaw(byte[] b, int offset, int length) + { + throw new RuntimeException("type error"); + } +} + diff --git a/java-plan2/src/org/msgpack/schema/ShortSchema.java b/java-plan2/src/org/msgpack/schema/ShortSchema.java new file mode 100644 index 0000000..aa95f51 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/ShortSchema.java @@ -0,0 +1,46 @@ +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class ShortSchema extends Schema { + public ShortSchema() + { + super("Short"); + } + + @Override + public String getExpression() + { + return "short"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + pk.packShort((Short)obj); + } + + @Override + public Object convert(GenericObject obj) + { + return obj.asShort(); + } + + @Override + public Object createByte(byte v) + { + return (int)v; + } + + @Override + public Object createShort(short v) + { + return (int)v; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/SpecificClassSchema.java b/java-plan2/src/org/msgpack/schema/SpecificClassSchema.java new file mode 100644 index 0000000..75c474a --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/SpecificClassSchema.java @@ -0,0 +1,156 @@ +package org.msgpack.schema; + +import java.util.List; +import java.util.Map; +import java.io.IOException; +import java.util.Iterator; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import org.msgpack.*; + +public class SpecificClassSchema extends ClassSchema { + private List fields; + private String namespace; + private List imports; + + private String fqdn; + private Constructor constructorCache; + + public SpecificClassSchema(String name, List fields, String namespace, List imports) + { + super(name, namespace, imports); + this.fields = fields; + if(namespace == null) { + this.fqdn = name; + } else { + this.fqdn = namespace+"."+name; + } + } + + //@Override + //public String getFullName() + //{ + // if(namespace == null) { + // return getName(); + // } else { + // return namespace+"."+getName(); + // } + //} + + public List getFields() + { + return fields; + } + + @Override + public String getExpression() + { + StringBuffer b = new StringBuffer(); + b.append("(class "); + b.append(getName()); + if(namespace != null) { + b.append(" (package "+namespace+")"); + } + for(SpecificFieldSchema f : fields) { + b.append(" "+f.getExpression()); + } + b.append(")"); + return b.toString(); + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + + if(constructorCache == null) { + cacheConstructor(); + } + + pk.packArray(fields.size()); + for(SpecificFieldSchema f : fields) { + f.getType().pack(pk, f.getFieldValue(obj)); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(GenericObject obj) + { + if(constructorCache == null) { + cacheConstructor(); + } + + List d = obj.asArray(); + + try { + Object g = constructorCache.newInstance((Object[])null); + + Iterator vi = d.iterator(); + Iterator fi = fields.iterator(); + while(fi.hasNext() && vi.hasNext()) { + SpecificFieldSchema f = fi.next(); + GenericObject v = vi.next(); + f.setFieldValue(g, f.getType().convert(v)); + } + // leave it as uninitialized + //while(fi.hasNext()) { + // SpecificFieldSchema f = fi.next(); + // g.put(f.getName(), null); + //} + + return g; + + } catch (InvocationTargetException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (InstantiationException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (IllegalAccessException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } + } + + private void cacheConstructor() + { + try { + Class c = Class.forName(fqdn); + int index = 0; + for(SpecificFieldSchema f : fields) { + f.cacheField(c, index++); + } + constructorCache = c.getDeclaredConstructor((Class[])null); + constructorCache.setAccessible(true); + } catch(ClassNotFoundException e) { + throw new RuntimeException("class not found: "+fqdn); + } catch (NoSuchMethodException e) { + throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); + } + } + + @Override + public Object newInstance() + { + if(constructorCache == null) { + cacheConstructor(); + } + try { + return constructorCache.newInstance((Object[])null); + } catch (InvocationTargetException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (InstantiationException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (IllegalAccessException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } + } + + public boolean equals(SpecificClassSchema o) + { + return (namespace != null ? namespace.equals(o.getNamespace()) : o.getNamespace() == null) && + getName().equals(o.getName()); + } +} + diff --git a/java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java b/java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java new file mode 100644 index 0000000..297df27 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java @@ -0,0 +1,78 @@ +package org.msgpack.schema; + +import java.util.Map; +import java.util.Arrays; +import java.lang.reflect.Field; +import org.msgpack.*; + +public class SpecificFieldSchema extends FieldSchema { + public Field fieldCache; + private int index; + + public SpecificFieldSchema(String name, Schema type) + { + super(name, type); + this.index = -1; + } + + @Override + public Object getFieldValue(Object obj) + { + if(index >= 0) { + return ((MessageMergeable)obj).getField(index); + } + + try { + return fieldCache.get(obj); + } catch(IllegalArgumentException e) { + throw new RuntimeException("can't get value from '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); + } catch(IllegalAccessException e) { + throw new RuntimeException("can't get value from '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); + } + } + + @Override + public void setFieldValue(Object obj, Object value) + { + if(index >= 0) { + ((MessageMergeable)obj).setField(index, value); + return; + } + + try { + fieldCache.set(obj, value); + } catch(IllegalArgumentException e) { + throw new RuntimeException("can't set value into '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); + } catch(IllegalAccessException e) { + throw new RuntimeException("can't set value into '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); + } + } + + void cacheField(Class c, int index) + { + for(Class i : c.getInterfaces()) { + if(i.equals(MessageMergeable.class)) { + this.index = index; + return; + } + } + + try { + fieldCache = c.getDeclaredField(getName()); + if(!fieldCache.isAccessible()) { + fieldCache.setAccessible(true); + } + } catch(NoSuchFieldException e) { + throw new RuntimeException("can't get '"+getName()+"' field of '"+c.getName()+"' class: "+e.getMessage()); + } catch(SecurityException e) { + throw new RuntimeException("can't get '"+getName()+"' field of '"+c.getName()+"' class: "+e.getMessage()); + } + } + + //public void setFieldInt(Object obj, int value) + //{ + // if(type instanceof PrimitiveSchema) { + // } + //} +} + diff --git a/java-plan2/src/org/msgpack/schema/StringSchema.java b/java-plan2/src/org/msgpack/schema/StringSchema.java new file mode 100644 index 0000000..fc6855b --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/StringSchema.java @@ -0,0 +1,48 @@ +package org.msgpack.schema; + +import java.io.IOException; +import java.nio.charset.Charset; +import org.msgpack.*; + +public class StringSchema extends Schema { + public StringSchema() + { + super("string"); + } + + public String getFullName() + { + return "String"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + String s = (String)obj; + byte[] d = s.getBytes("UTF-8"); + pk.packRaw(d.length); + pk.packRawBody(d); + } + + @Override + public Object convert(GenericObject obj) + { + return obj.asString(); + } + + @Override + public Object createRaw(byte[] b, int offset, int length) + { + try { + return new String(b, offset, length, "UTF-8"); // XXX FIXME debug + } catch (Exception e) { + // FIXME + throw new RuntimeException(e.getMessage()); + } + } +} + diff --git a/java-plan2/test/Generate.java b/java-plan2/test/Generate.java new file mode 100644 index 0000000..2ac5878 --- /dev/null +++ b/java-plan2/test/Generate.java @@ -0,0 +1,18 @@ +import java.io.*; +import java.util.*; +import org.msgpack.*; +import org.msgpack.schema.*; + +public class Generate { + public static void main(String[] args) throws IOException + { + Writer output = new OutputStreamWriter(System.out); + + Schema s1 = Schema.parse("(class Test (field uri raw) (field width int))"); + ClassGenerator.write(s1, output); + + Schema s1 = Schema.parse("(class MediaContent (package serializers.msgpack) (field image (array (class Image (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); + ClassGenerator.write(s2, output); + } +} + diff --git a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java new file mode 100644 index 0000000..ecb64fd --- /dev/null +++ b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java @@ -0,0 +1,277 @@ +package serializers.msgpack; + +import java.util.*; +import java.io.*; +import org.msgpack.*; +import org.msgpack.schema.*; + +public final class MediaContent implements MessagePackable, MessageConvertable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class MediaContent (package serializers.msgpack) (field image (array (class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public ArrayList image; + public Media media; + + public MediaContent() { } + + @Override + public void messagePack(Packer pk) throws IOException + { + List _f = _SCHEMA.getFields(); + pk.packArray(2); + _f.get(0).getType().pack(pk, image); + _f.get(1).getType().pack(pk, media); + } + + @Override + @SuppressWarnings("unchecked") + public void messageConvert(GenericObject obj) + { + List _l = obj.asArray(); + List _f = _SCHEMA.getFields(); + if(_l.size() <= 0) { return; } image = (ArrayList)_f.get(0).getType().convert(_l.get(0)); + if(_l.size() <= 1) { return; } media = (Media)_f.get(1).getType().convert(_l.get(1)); + } + + public static MediaContent convert(GenericObject obj) + { + return (MediaContent)_SCHEMA.convert(obj); + } + + public void setField(int index, Object value) + { + switch(index) { + case 0: + image = (ArrayList)value; + break; + case 1: + media = (Media)value; + break; + } + } + + public Object getField(int index) + { + switch(index) { + case 0: + return image; + case 1: + return media; + } + return null; + } +} + +final class Image implements MessagePackable, MessageConvertable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public String uri; + public String title; + public Integer width; + public Integer height; + public Integer size; + + public Image() { } + + @Override + public void messagePack(Packer pk) throws IOException + { + List _f = _SCHEMA.getFields(); + pk.packArray(5); + _f.get(0).getType().pack(pk, uri); + _f.get(1).getType().pack(pk, title); + _f.get(2).getType().pack(pk, width); + _f.get(3).getType().pack(pk, height); + _f.get(4).getType().pack(pk, size); + } + + @Override + @SuppressWarnings("unchecked") + public void messageConvert(GenericObject obj) + { + List _l = obj.asArray(); + List _f = _SCHEMA.getFields(); + if(_l.size() <= 0) { return; } uri = (String)_f.get(0).getType().convert(_l.get(0)); + if(_l.size() <= 1) { return; } title = (String)_f.get(1).getType().convert(_l.get(1)); + if(_l.size() <= 2) { return; } width = (Integer)_f.get(2).getType().convert(_l.get(2)); + if(_l.size() <= 3) { return; } height = (Integer)_f.get(3).getType().convert(_l.get(3)); + if(_l.size() <= 4) { return; } size = (Integer)_f.get(4).getType().convert(_l.get(4)); + } + + public static Image convert(GenericObject obj) + { + return (Image)_SCHEMA.convert(obj); + } + + public void setField(int index, Object value) + { + switch(index) { + case 0: + uri = (String)value; + break; + case 1: + title = (String)value; + break; + case 2: + width = (Integer)value; + break; + case 3: + height = (Integer)value; + break; + case 4: + size = (Integer)value; + break; + } + } + + public Object getField(int index) + { + switch(index) { + case 0: + return uri; + case 1: + return title; + case 2: + return width; + case 3: + return height; + case 4: + return size; + } + return null; + } +} + +final class Media implements MessagePackable, MessageConvertable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public String uri; + public String title; + public Integer width; + public Integer height; + public String format; + public Long duration; + public Long size; + public Integer bitrate; + public ArrayList person; + public Integer player; + public String copyright; + + public Media() { } + + @Override + public void messagePack(Packer pk) throws IOException + { + List _f = _SCHEMA.getFields(); + pk.packArray(11); + _f.get(0).getType().pack(pk, uri); + _f.get(1).getType().pack(pk, title); + _f.get(2).getType().pack(pk, width); + _f.get(3).getType().pack(pk, height); + _f.get(4).getType().pack(pk, format); + _f.get(5).getType().pack(pk, duration); + _f.get(6).getType().pack(pk, size); + _f.get(7).getType().pack(pk, bitrate); + _f.get(8).getType().pack(pk, person); + _f.get(9).getType().pack(pk, player); + _f.get(10).getType().pack(pk, copyright); + } + + @Override + @SuppressWarnings("unchecked") + public void messageConvert(GenericObject obj) + { + List _l = obj.asArray(); + List _f = _SCHEMA.getFields(); + if(_l.size() <= 0) { return; } uri = (String)_f.get(0).getType().convert(_l.get(0)); + if(_l.size() <= 1) { return; } title = (String)_f.get(1).getType().convert(_l.get(1)); + if(_l.size() <= 2) { return; } width = (Integer)_f.get(2).getType().convert(_l.get(2)); + if(_l.size() <= 3) { return; } height = (Integer)_f.get(3).getType().convert(_l.get(3)); + if(_l.size() <= 4) { return; } format = (String)_f.get(4).getType().convert(_l.get(4)); + if(_l.size() <= 5) { return; } duration = (Long)_f.get(5).getType().convert(_l.get(5)); + if(_l.size() <= 6) { return; } size = (Long)_f.get(6).getType().convert(_l.get(6)); + if(_l.size() <= 7) { return; } bitrate = (Integer)_f.get(7).getType().convert(_l.get(7)); + if(_l.size() <= 8) { return; } person = (ArrayList)_f.get(8).getType().convert(_l.get(8)); + if(_l.size() <= 9) { return; } player = (Integer)_f.get(9).getType().convert(_l.get(9)); + if(_l.size() <= 10) { return; } copyright = (String)_f.get(10).getType().convert(_l.get(10)); + } + + public static Media convert(GenericObject obj) + { + return (Media)_SCHEMA.convert(obj); + } + + public void setField(int index, Object value) + { + switch(index) { + case 0: + uri = (String)value; + break; + case 1: + title = (String)value; + break; + case 2: + width = (Integer)value; + break; + case 3: + height = (Integer)value; + break; + case 4: + format = (String)value; + break; + case 5: + duration = (Long)value; + break; + case 6: + size = (Long)value; + break; + case 7: + bitrate = (Integer)value; + break; + case 8: + person = (ArrayList)value; + break; + case 9: + player = (Integer)value; + break; + case 10: + copyright = (String)value; + break; + } + } + + public Object getField(int index) + { + switch(index) { + case 0: + return uri; + case 1: + return title; + case 2: + return width; + case 3: + return height; + case 4: + return format; + case 5: + return duration; + case 6: + return size; + case 7: + return bitrate; + case 8: + return person; + case 9: + return player; + case 10: + return copyright; + } + return null; + } + +} diff --git a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs new file mode 100644 index 0000000..547ba48 --- /dev/null +++ b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs @@ -0,0 +1,21 @@ +(class MediaContent + (package serializers.msgpack) + (field image (array (class Image + (field uri string) + (field title string) + (field width int) + (field height int) + (field size int)))) + (field media (class Media + (field uri string) + (field title string) + (field width int) + (field height int) + (field format string) + (field duration long) + (field size long) + (field bitrate int) + (field person (array string)) + (field player int) + (field copyright string))) + ) diff --git a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java new file mode 100644 index 0000000..acb5580 --- /dev/null +++ b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java @@ -0,0 +1,70 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import java.nio.charset.Charset; + +import org.msgpack.*; +import org.msgpack.schema.*; +import serializers.msgpack.*; + +import serializers.ObjectSerializer; + +public class MessagePackSerializer implements ObjectSerializer +{ + public String getName() { + return "msgpack-specific"; + } + + public MediaContent create() throws Exception { + Media media = new Media(); + media.uri = "http://javaone.com/keynote.mpg"; + media.format = "video/mpg4"; + media.title = "Javaone Keynote"; + media.duration = 1234567L; + media.bitrate = 0; + media.person = new ArrayList(2); + media.person.add("Bill Gates"); + media.person.add("Steve Jobs"); + media.player = 0; + media.height = 0; + media.width = 0; + media.size = 123L; + media.copyright = ""; + + Image image1 = new Image(); + image1.uri = "http://javaone.com/keynote_large.jpg"; + image1.width = 0; + image1.height = 0; + image1.size = 2; + image1.title = "Javaone Keynote"; + + Image image2 = new Image(); + image2.uri = "http://javaone.com/keynote_thumbnail.jpg"; + image2.width = 0; + image2.height = 0; + image2.size = 1; + image2.title = "Javaone Keynote"; + + MediaContent content = new MediaContent(); + content.media = media; + content.image = new ArrayList(2); + content.image.add(image1); + content.image.add(image2); + return content; + } + + public MediaContent deserialize(byte[] array) throws Exception { + UnbufferedUnpacker pac = new UnbufferedUnpacker().useSchema(MediaContent.getSchema()); + pac.execute(array); + return (MediaContent)pac.getData(); + } + + public byte[] serialize(MediaContent content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.pack(content); + return os.toByteArray(); + } +} + From eb9e89249137cbed0ace62de9902e69bd082b2a3 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 26 Nov 2009 11:22:08 +0900 Subject: [PATCH 0314/1172] import MessagePack for Java implementation plan 3 --- java-plan2/test/Generate.java | 2 +- java-plan3/build.xml | 15 + .../src/org/msgpack/MessageMergeable.java | 23 + .../src/org/msgpack/MessagePackable.java | 25 + .../src/org/msgpack/MessageTypeException.java | 39 ++ java-plan3/src/org/msgpack/Packer.java | 409 +++++++++++++++ java-plan3/src/org/msgpack/Schema.java | 133 +++++ .../src/org/msgpack/UnbufferedUnpacker.java | 82 +++ .../src/org/msgpack/UnpackException.java | 29 ++ .../src/org/msgpack/UnpackIterator.java | 66 +++ java-plan3/src/org/msgpack/Unpacker.java | 245 +++++++++ .../src/org/msgpack/impl/UnpackerImpl.java | 483 ++++++++++++++++++ .../src/org/msgpack/schema/ArraySchema.java | 125 +++++ .../src/org/msgpack/schema/ByteSchema.java | 89 ++++ .../org/msgpack/schema/ClassGenerator.java | 241 +++++++++ .../src/org/msgpack/schema/ClassSchema.java | 95 ++++ .../src/org/msgpack/schema/DoubleSchema.java | 84 +++ .../src/org/msgpack/schema/FieldSchema.java | 43 ++ .../src/org/msgpack/schema/FloatSchema.java | 84 +++ .../msgpack/schema/GenericClassSchema.java | 91 ++++ .../src/org/msgpack/schema/GenericSchema.java | 192 +++++++ .../src/org/msgpack/schema/IArraySchema.java | 26 + .../src/org/msgpack/schema/IMapSchema.java | 27 + .../src/org/msgpack/schema/IntSchema.java | 89 ++++ .../src/org/msgpack/schema/LongSchema.java | 89 ++++ .../src/org/msgpack/schema/MapSchema.java | 102 ++++ .../src/org/msgpack/schema/RawSchema.java | 105 ++++ .../msgpack/schema/ReflectionClassSchema.java | 64 +++ .../src/org/msgpack/schema/SSchemaParser.java | 254 +++++++++ .../src/org/msgpack/schema/ShortSchema.java | 89 ++++ .../msgpack/schema/SpecificClassSchema.java | 122 +++++ .../src/org/msgpack/schema/StringSchema.java | 111 ++++ java-plan3/test/Generate.java | 38 ++ .../src/serializers/msgpack/MediaContent.java | 173 +++++++ .../serializers/msgpack/MediaContent.mpacs | 21 + .../msgpack/MessagePackDynamicSerializer.java | 68 +++ .../msgpack/MessagePackGenericSerializer.java | 70 +++ .../MessagePackIndirectSerializer.java | 67 +++ .../MessagePackSpecificSerializer.java | 66 +++ 39 files changed, 4175 insertions(+), 1 deletion(-) create mode 100644 java-plan3/build.xml create mode 100644 java-plan3/src/org/msgpack/MessageMergeable.java create mode 100644 java-plan3/src/org/msgpack/MessagePackable.java create mode 100644 java-plan3/src/org/msgpack/MessageTypeException.java create mode 100644 java-plan3/src/org/msgpack/Packer.java create mode 100644 java-plan3/src/org/msgpack/Schema.java create mode 100644 java-plan3/src/org/msgpack/UnbufferedUnpacker.java create mode 100644 java-plan3/src/org/msgpack/UnpackException.java create mode 100644 java-plan3/src/org/msgpack/UnpackIterator.java create mode 100644 java-plan3/src/org/msgpack/Unpacker.java create mode 100644 java-plan3/src/org/msgpack/impl/UnpackerImpl.java create mode 100644 java-plan3/src/org/msgpack/schema/ArraySchema.java create mode 100644 java-plan3/src/org/msgpack/schema/ByteSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/ClassGenerator.java create mode 100644 java-plan3/src/org/msgpack/schema/ClassSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/DoubleSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/FieldSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/FloatSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/GenericClassSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/GenericSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/IArraySchema.java create mode 100644 java-plan3/src/org/msgpack/schema/IMapSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/IntSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/LongSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/MapSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/RawSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/SSchemaParser.java create mode 100644 java-plan3/src/org/msgpack/schema/ShortSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/SpecificClassSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/StringSchema.java create mode 100644 java-plan3/test/Generate.java create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java diff --git a/java-plan2/test/Generate.java b/java-plan2/test/Generate.java index 2ac5878..6b7800b 100644 --- a/java-plan2/test/Generate.java +++ b/java-plan2/test/Generate.java @@ -11,7 +11,7 @@ public class Generate { Schema s1 = Schema.parse("(class Test (field uri raw) (field width int))"); ClassGenerator.write(s1, output); - Schema s1 = Schema.parse("(class MediaContent (package serializers.msgpack) (field image (array (class Image (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); + Schema s2 = Schema.parse("(class MediaContent (package serializers.msgpack) (field image (array (class Image (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); ClassGenerator.write(s2, output); } } diff --git a/java-plan3/build.xml b/java-plan3/build.xml new file mode 100644 index 0000000..598a853 --- /dev/null +++ b/java-plan3/build.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/java-plan3/src/org/msgpack/MessageMergeable.java b/java-plan3/src/org/msgpack/MessageMergeable.java new file mode 100644 index 0000000..e11119c --- /dev/null +++ b/java-plan3/src/org/msgpack/MessageMergeable.java @@ -0,0 +1,23 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +public interface MessageMergeable { + public void messageMerge(Object obj) throws MessageTypeException; +} + diff --git a/java-plan3/src/org/msgpack/MessagePackable.java b/java-plan3/src/org/msgpack/MessagePackable.java new file mode 100644 index 0000000..d8a7db9 --- /dev/null +++ b/java-plan3/src/org/msgpack/MessagePackable.java @@ -0,0 +1,25 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.IOException; + +public interface MessagePackable { + public void messagePack(Packer pk) throws IOException; +} + diff --git a/java-plan3/src/org/msgpack/MessageTypeException.java b/java-plan3/src/org/msgpack/MessageTypeException.java new file mode 100644 index 0000000..09031b2 --- /dev/null +++ b/java-plan3/src/org/msgpack/MessageTypeException.java @@ -0,0 +1,39 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.IOException; + +public class MessageTypeException extends IOException { + public MessageTypeException() { } + + public MessageTypeException(String s) { + super(s); + } + + public static MessageTypeException invalidConvert(Object from, Schema to) { + return new MessageTypeException(from.getClass().getName()+" cannot be convert to "+to.getExpression()); + } + + /* FIXME + public static MessageTypeException schemaMismatch(Schema to) { + return new MessageTypeException("schema mismatch "+to.getExpression()); + } + */ +} + diff --git a/java-plan3/src/org/msgpack/Packer.java b/java-plan3/src/org/msgpack/Packer.java new file mode 100644 index 0000000..7f2508c --- /dev/null +++ b/java-plan3/src/org/msgpack/Packer.java @@ -0,0 +1,409 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.OutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Map; + +public class Packer { + protected byte[] castBytes = new byte[9]; + protected ByteBuffer castBuffer = ByteBuffer.wrap(castBytes); + protected OutputStream out; + + public Packer(OutputStream out) { + this.out = out; + } + + public Packer packByte(byte d) throws IOException { + if(d < -(1<<5)) { + castBytes[0] = (byte)0xd1; + castBytes[1] = d; + out.write(castBytes, 0, 2); + } else { + out.write(d); + } + return this; + } + + public Packer packShort(short d) throws IOException { + if(d < -(1<<5)) { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } + } + return this; + } + + public Packer packInt(int d) throws IOException { + if(d < -(1<<5)) { + if(d < -(1<<15)) { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } else if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else if(d < (1<<16)) { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } + } + return this; + } + + public Packer packLong(long d) throws IOException { + if(d < -(1L<<5)) { + if(d < -(1L<<15)) { + if(d < -(1L<<31)) { + // signed 64 + castBytes[0] = (byte)0xd3; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } else { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } + } else { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1L<<16)) { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + //System.out.println("pack uint 16 "+(short)d); + } + } else { + if(d < (1L<<32)) { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } else { + // unsigned 64 + castBytes[0] = (byte)0xcf; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } + } + } + return this; + } + + public Packer packFloat(float d) throws IOException { + castBytes[0] = (byte)0xca; + castBuffer.putFloat(1, d); + out.write(castBytes, 0, 5); + return this; + } + + public Packer packDouble(double d) throws IOException { + castBytes[0] = (byte)0xcb; + castBuffer.putDouble(1, d); + out.write(castBytes, 0, 9); + return this; + } + + public Packer packNil() throws IOException { + out.write((byte)0xc0); + return this; + } + + public Packer packTrue() throws IOException { + out.write((byte)0xc3); + return this; + } + + public Packer packFalse() throws IOException { + out.write((byte)0xc2); + return this; + } + + public Packer packArray(int n) throws IOException { + if(n < 16) { + final int d = 0x90 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xdc; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdd; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packMap(int n) throws IOException { + if(n < 16) { + final int d = 0x80 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xde; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdf; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRaw(int n) throws IOException { + if(n < 32) { + final int d = 0xa0 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xda; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdb; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRawBody(byte[] b) throws IOException { + out.write(b); + return this; + } + + public Packer packRawBody(byte[] b, int off, int length) throws IOException { + out.write(b, off, length); + return this; + } + + + public Packer packWithSchema(Object o, Schema s) throws IOException { + s.pack(this, o); + return this; + } + + + public Packer packString(String s) throws IOException { + byte[] b = ((String)s).getBytes("UTF-8"); + packRaw(b.length); + return packRawBody(b); + } + + + public Packer pack(String o) throws IOException { + if(o == null) { return packNil(); } + return packString(o); + } + + public Packer pack(MessagePackable o) throws IOException { + if(o == null) { return packNil(); } + o.messagePack(this); + return this; + } + + public Packer pack(byte[] o) throws IOException { + if(o == null) { return packNil(); } + packRaw(o.length); + return packRawBody(o); + } + + public Packer pack(List o) throws IOException { + if(o == null) { return packNil(); } + packArray(o.size()); + for(Object i : o) { pack(i); } + return this; + } + + @SuppressWarnings("unchecked") + public Packer pack(Map o) throws IOException { + if(o == null) { return packNil(); } + packMap(o.size()); + for(Map.Entry e : ((Map)o).entrySet()) { + pack(e.getKey()); + pack(e.getValue()); + } + return this; + } + + public Packer pack(Boolean o) throws IOException { + if(o == null) { return packNil(); } + if(o) { + return packTrue(); + } else { + return packFalse(); + } + } + + public Packer pack(Byte o) throws IOException { + if(o == null) { return packNil(); } + return packByte(o); + } + + public Packer pack(Short o) throws IOException { + if(o == null) { return packNil(); } + return packShort(o); + } + + public Packer pack(Integer o) throws IOException { + if(o == null) { return packNil(); } + return packInt(o); + } + + public Packer pack(Long o) throws IOException { + if(o == null) { return packNil(); } + return packLong(o); + } + + public Packer pack(Float o) throws IOException { + if(o == null) { return packNil(); } + return packFloat(o); + } + + public Packer pack(Double o) throws IOException { + if(o == null) { return packNil(); } + return packDouble(o); + } + + + @SuppressWarnings("unchecked") + public Packer pack(Object o) throws IOException { + if(o == null) { + return packNil(); + } else if(o instanceof String) { + byte[] b = ((String)o).getBytes("UTF-8"); + packRaw(b.length); + return packRawBody(b); + } else if(o instanceof MessagePackable) { + ((MessagePackable)o).messagePack(this); + return this; + } else if(o instanceof byte[]) { + byte[] b = (byte[])o; + packRaw(b.length); + return packRawBody(b); + } else if(o instanceof List) { + List l = (List)o; + packArray(l.size()); + for(Object i : l) { pack(i); } + return this; + } else if(o instanceof Map) { + Map m = (Map)o; + packMap(m.size()); + for(Map.Entry e : m.entrySet()) { + pack(e.getKey()); + pack(e.getValue()); + } + return this; + } else if(o instanceof Boolean) { + if((Boolean)o) { + return packTrue(); + } else { + return packFalse(); + } + } else if(o instanceof Integer) { + return packInt((Integer)o); + } else if(o instanceof Long) { + return packLong((Long)o); + } else if(o instanceof Short) { + return packShort((Short)o); + } else if(o instanceof Byte) { + return packByte((Byte)o); + } else if(o instanceof Float) { + return packFloat((Float)o); + } else if(o instanceof Double) { + return packDouble((Double)o); + } else { + throw new IOException("unknown object "+o+" ("+o.getClass()+")"); + } + } +} + diff --git a/java-plan3/src/org/msgpack/Schema.java b/java-plan3/src/org/msgpack/Schema.java new file mode 100644 index 0000000..f99b3d0 --- /dev/null +++ b/java-plan3/src/org/msgpack/Schema.java @@ -0,0 +1,133 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.Writer; +import java.io.IOException; +import org.msgpack.schema.SSchemaParser; +import org.msgpack.schema.ClassGenerator; + +public abstract class Schema { + private String expression; + private String name; + + public Schema(String name) { + this.expression = expression; + this.name = name; + } + + public String getName() { + return name; + } + + public String getFullName() { + return name; + } + + public String getExpression() { + return name; + } + + public static Schema parse(String source) { + return SSchemaParser.parse(source); + } + + public static Schema load(String source) { + return SSchemaParser.load(source); + } + + public void write(Writer output) throws IOException { + ClassGenerator.write(this, output); + } + + public abstract void pack(Packer pk, Object obj) throws IOException; + + public abstract Object convert(Object obj) throws MessageTypeException; + + + public Object createFromNil() { + return null; + } + + public Object createFromBoolean(boolean v) { + throw new RuntimeException("type error"); + } + + public Object createFromByte(byte v) { + throw new RuntimeException("type error"); + } + + public Object createFromShort(short v) { + throw new RuntimeException("type error"); + } + + public Object createFromInt(int v) { + throw new RuntimeException("type error"); + } + + public Object createFromLong(long v) { + throw new RuntimeException("type error"); + } + + public Object createFromFloat(float v) { + throw new RuntimeException("type error"); + } + + public Object createFromDouble(double v) { + throw new RuntimeException("type error"); + } + + public Object createFromRaw(byte[] b, int offset, int length) { + throw new RuntimeException("type error"); + } + + /* FIXME + public Object createFromBoolean(boolean v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromByte(byte v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromShort(short v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromInt(int v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromLong(long v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromFloat(float v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromDouble(double v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromRaw(byte[] b, int offset, int length) { + throw MessageTypeException.schemaMismatch(this); + } + */ +} + diff --git a/java-plan3/src/org/msgpack/UnbufferedUnpacker.java b/java-plan3/src/org/msgpack/UnbufferedUnpacker.java new file mode 100644 index 0000000..471605f --- /dev/null +++ b/java-plan3/src/org/msgpack/UnbufferedUnpacker.java @@ -0,0 +1,82 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.lang.Iterable; +import java.io.InputStream; +import java.io.IOException; +import java.util.Iterator; +import org.msgpack.impl.UnpackerImpl; + +public class UnbufferedUnpacker extends UnpackerImpl { + private int offset; + private boolean finished; + private Object data; + + public UnbufferedUnpacker() { + super(); + this.offset = 0; + this.finished = false; + } + + public UnbufferedUnpacker useSchema(Schema s) { + super.setSchema(s); + return this; + } + + public Object getData() { + return data; + } + + public boolean isFinished() { + return finished; + } + + public void reset() { + super.reset(); + this.offset = 0; + } + + int getOffset() { + return offset; + } + + void setOffset(int offset) { + this.offset = offset; + } + + public int execute(byte[] buffer) throws UnpackException { + return execute(buffer, 0, buffer.length); + } + + // FIXME + public int execute(byte[] buffer, int offset, int length) throws UnpackException + { + int noffset = super.execute(buffer, offset + this.offset, length); + this.offset = noffset - offset; + if(super.isFinished()) { + this.data = super.getData(); + this.finished = true; + super.reset(); + } else { + this.finished = false; + } + return noffset; + } +} + diff --git a/java-plan3/src/org/msgpack/UnpackException.java b/java-plan3/src/org/msgpack/UnpackException.java new file mode 100644 index 0000000..db08d95 --- /dev/null +++ b/java-plan3/src/org/msgpack/UnpackException.java @@ -0,0 +1,29 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.IOException; + +public class UnpackException extends IOException { + public UnpackException() { } + + public UnpackException(String s) { + super(s); + } +} + diff --git a/java-plan3/src/org/msgpack/UnpackIterator.java b/java-plan3/src/org/msgpack/UnpackIterator.java new file mode 100644 index 0000000..9975b68 --- /dev/null +++ b/java-plan3/src/org/msgpack/UnpackIterator.java @@ -0,0 +1,66 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.IOException; +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class UnpackIterator implements Iterator { + private Unpacker pac; + private boolean have; + private Object data; + + UnpackIterator(Unpacker pac) { + this.pac = pac; + this.have = false; + } + + public boolean hasNext() { + if(have) { return true; } + try { + while(true) { + if(pac.execute()) { + data = pac.getData(); + pac.reset(); + have = true; + return true; + } + + if(!pac.fill()) { + return false; + } + } + } catch (IOException e) { + return false; + } + } + + public Object next() { + if(!have && !hasNext()) { + throw new NoSuchElementException(); + } + have = false; + return data; + } + + public void remove() { + throw new UnsupportedOperationException(); + } +} + diff --git a/java-plan3/src/org/msgpack/Unpacker.java b/java-plan3/src/org/msgpack/Unpacker.java new file mode 100644 index 0000000..af211c6 --- /dev/null +++ b/java-plan3/src/org/msgpack/Unpacker.java @@ -0,0 +1,245 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.lang.Iterable; +import java.io.InputStream; +import java.io.IOException; +import java.util.Iterator; +import org.msgpack.impl.UnpackerImpl; + +public class Unpacker extends UnpackerImpl implements Iterable { + + public static final int DEFAULT_BUFFER_SIZE = 32*1024; + + private int used; + private int offset; + private int parsed; + private byte[] buffer; + private int bufferReserveSize; + private InputStream stream; + + public Unpacker() { + this(DEFAULT_BUFFER_SIZE); + } + + public Unpacker(int bufferReserveSize) { + this(null, bufferReserveSize); + } + + public Unpacker(InputStream stream) { + this(stream, DEFAULT_BUFFER_SIZE); + } + + public Unpacker(InputStream stream, int bufferReserveSize) { + super(); + this.used = 0; + this.offset = 0; + this.parsed = 0; + this.buffer = new byte[bufferReserveSize]; + this.bufferReserveSize = bufferReserveSize/2; + this.stream = stream; + } + + public Unpacker useSchema(Schema s) { + super.setSchema(s); + return this; + } + + public void reserveBuffer(int size) { + if(buffer.length - used >= size) { + return; + } + /* + if(used == parsed && buffer.length >= size) { + // rewind buffer + used = 0; + offset = 0; + return; + } + */ + + int nextSize = buffer.length * 2; + while(nextSize < size + used) { + nextSize *= 2; + } + + byte[] tmp = new byte[nextSize]; + System.arraycopy(buffer, offset, tmp, 0, used - offset); + + buffer = tmp; + used -= offset; + offset = 0; + } + + public byte[] getBuffer() { + return buffer; + } + + public int getBufferOffset() { + return used; + } + + public int getBufferCapacity() { + return buffer.length - used; + } + + public void bufferConsumed(int size) { + used += size; + } + + public void feed(byte[] buffer) { + feed(buffer, 0, buffer.length); + } + + public void feed(byte[] buffer, int offset, int length) { + reserveBuffer(length); + System.arraycopy(buffer, offset, this.buffer, this.offset, length); + bufferConsumed(length); + } + + public boolean fill() throws IOException { + if(stream == null) { + return false; + } + reserveBuffer(bufferReserveSize); + int rl = stream.read(getBuffer(), getBufferOffset(), getBufferCapacity()); + if(rl <= 0) { + return false; + } + bufferConsumed(rl); + return true; + } + + public Iterator iterator() { + return new UnpackIterator(this); + } + + public boolean execute() throws UnpackException { + int noffset = super.execute(buffer, offset, used); + if(noffset <= offset) { + return false; + } + parsed += noffset - offset; + offset = noffset; + return super.isFinished(); + } + + public Object getData() { + return super.getData(); + } + + public void reset() { + super.reset(); + parsed = 0; + } + + public int getMessageSize() { + return parsed - offset + used; + } + + public int getParsedSize() { + return parsed; + } + + public int getNonParsedSize() { + return used - offset; + } + + public void skipNonparsedBuffer(int size) { + offset += size; + } + + public void removeNonparsedBuffer() { + used = offset; + } + + /* + public static class Context { + private boolean finished; + private Object data; + private int offset; + private UnpackerImpl impl; + + public Context() + { + this.finished = false; + this.impl = new UnpackerImpl(); + } + + public boolean isFinished() + { + return finished; + } + + public Object getData() + { + return data; + } + + int getOffset() + { + return offset; + } + + void setFinished(boolean finished) + { + this.finished = finished; + } + + void setData(Object data) + { + this.data = data; + } + + void setOffset(int offset) + { + this.offset = offset; + } + + UnpackerImpl getImpl() + { + return impl; + } + } + + public static int unpack(Context ctx, byte[] buffer) throws UnpackException + { + return unpack(ctx, buffer, 0, buffer.length); + } + + public static int unpack(Context ctx, byte[] buffer, int offset, int length) throws UnpackException + { + UnpackerImpl impl = ctx.getImpl(); + int noffset = impl.execute(buffer, offset + ctx.getOffset(), length); + ctx.setOffset(noffset - offset); + if(impl.isFinished()) { + ctx.setData(impl.getData()); + ctx.setFinished(false); + impl.reset(); + } else { + ctx.setData(null); + ctx.setFinished(true); + } + int parsed = noffset - offset; + ctx.setOffset(parsed); + return noffset; + } + */ +} + diff --git a/java-plan3/src/org/msgpack/impl/UnpackerImpl.java b/java-plan3/src/org/msgpack/impl/UnpackerImpl.java new file mode 100644 index 0000000..47a1800 --- /dev/null +++ b/java-plan3/src/org/msgpack/impl/UnpackerImpl.java @@ -0,0 +1,483 @@ +package org.msgpack.impl; + +import java.nio.ByteBuffer; +//import java.math.BigInteger; +import org.msgpack.*; +import org.msgpack.schema.GenericSchema; +import org.msgpack.schema.IMapSchema; +import org.msgpack.schema.IArraySchema; + +public class UnpackerImpl { + static final int CS_HEADER = 0x00; + static final int CS_FLOAT = 0x0a; + static final int CS_DOUBLE = 0x0b; + static final int CS_UINT_8 = 0x0c; + static final int CS_UINT_16 = 0x0d; + static final int CS_UINT_32 = 0x0e; + static final int CS_UINT_64 = 0x0f; + static final int CS_INT_8 = 0x10; + static final int CS_INT_16 = 0x11; + static final int CS_INT_32 = 0x12; + static final int CS_INT_64 = 0x13; + static final int CS_RAW_16 = 0x1a; + static final int CS_RAW_32 = 0x1b; + static final int CS_ARRAY_16 = 0x1c; + static final int CS_ARRAY_32 = 0x1d; + static final int CS_MAP_16 = 0x1e; + static final int CS_MAP_32 = 0x1f; + static final int ACS_RAW_VALUE = 0x20; + static final int CT_ARRAY_ITEM = 0x00; + static final int CT_MAP_KEY = 0x01; + static final int CT_MAP_VALUE = 0x02; + + static final int MAX_STACK_SIZE = 16; + + private int cs; + private int trail; + private int top; + private int[] stack_ct = new int[MAX_STACK_SIZE]; + private int[] stack_count = new int[MAX_STACK_SIZE]; + private Object[] stack_obj = new Object[MAX_STACK_SIZE]; + private Schema[] stack_schema = new Schema[MAX_STACK_SIZE]; + private int top_ct; + private int top_count; + private Object top_obj; + private Schema top_schema; + private ByteBuffer castBuffer = ByteBuffer.allocate(8); + private boolean finished = false; + private Object data = null; + + private static final Schema GENERIC_SCHEMA = new GenericSchema(); + private Schema rootSchema; + + protected UnpackerImpl() + { + setSchema(GENERIC_SCHEMA); + } + + protected void setSchema(Schema schema) + { + this.rootSchema = schema; + reset(); + } + + protected Object getData() + { + return data; + } + + protected boolean isFinished() + { + return finished; + } + + protected void reset() + { + cs = CS_HEADER; + top = -1; + finished = false; + data = null; + top_ct = 0; + top_count = 0; + top_obj = null; + top_schema = rootSchema; + } + + @SuppressWarnings("unchecked") + protected int execute(byte[] src, int off, int length) throws UnpackException + { + if(off >= length) { return off; } + + int limit = length; + int i = off; + int count; + + Object obj = null; + + _out: do { + _header_again: { + //System.out.println("while i:"+i+" limit:"+limit); + + int b = src[i]; + + _push: { + _fixed_trail_again: + if(cs == CS_HEADER) { + + if((b & 0x80) == 0) { // Positive Fixnum + //System.out.println("positive fixnum "+b); + obj = top_schema.createFromByte((byte)b); + break _push; + } + + if((b & 0xe0) == 0xe0) { // Negative Fixnum + //System.out.println("negative fixnum "+b); + obj = top_schema.createFromByte((byte)b); + break _push; + } + + if((b & 0xe0) == 0xa0) { // FixRaw + trail = b & 0x1f; + if(trail == 0) { + obj = top_schema.createFromRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + } + + if((b & 0xf0) == 0x90) { // FixArray + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IArraySchema)) { + throw new RuntimeException("type error"); + } + count = b & 0x0f; + //System.out.println("fixarray count:"+count); + obj = new Object[count]; + if(count == 0) { break _push; } // FIXME check IArraySchema + ++top; + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + top_obj = obj; + top_ct = CT_ARRAY_ITEM; + top_count = count; + top_schema = ((IArraySchema)top_schema).getElementSchema(0); + break _header_again; + } + + if((b & 0xf0) == 0x80) { // FixMap + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IMapSchema)) { + throw new RuntimeException("type error"); + } + count = b & 0x0f; + obj = new Object[count*2]; + if(count == 0) { break _push; } // FIXME check IMapSchema + //System.out.println("fixmap count:"+count); + ++top; + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + top_obj = obj; + top_ct = CT_MAP_KEY; + top_count = count; + top_schema = ((IMapSchema)top_schema).getKeySchema(); + break _header_again; + } + + switch(b & 0xff) { // FIXME + case 0xc0: // nil + obj = top_schema.createFromNil(); + break _push; + case 0xc2: // false + obj = top_schema.createFromBoolean(false); + break _push; + case 0xc3: // true + obj = top_schema.createFromBoolean(true); + break _push; + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + trail = 1 << (b & 0x03); + cs = b & 0x1f; + //System.out.println("a trail "+trail+" cs:"+cs); + break _fixed_trail_again; + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + trail = 2 << (b & 0x01); + cs = b & 0x1f; + //System.out.println("b trail "+trail+" cs:"+cs); + break _fixed_trail_again; + default: + //System.out.println("unknown b "+(b&0xff)); + throw new UnpackException("parse error"); + } + + } // _fixed_trail_again + + do { + _fixed_trail_again: { + + if(limit - i <= trail) { break _out; } + int n = i + 1; + i += trail; + + switch(cs) { + case CS_FLOAT: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = top_schema.createFromFloat( castBuffer.getFloat(0) ); + //System.out.println("float "+obj); + break _push; + case CS_DOUBLE: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + obj = top_schema.createFromDouble( castBuffer.getDouble(0) ); + //System.out.println("double "+obj); + break _push; + case CS_UINT_8: + //System.out.println(n); + //System.out.println(src[n]); + //System.out.println(src[n+1]); + //System.out.println(src[n-1]); + obj = top_schema.createFromShort( (short)((src[n]) & 0xff) ); + //System.out.println("uint8 "+obj); + break _push; + case CS_UINT_16: + //System.out.println(src[n]); + //System.out.println(src[n+1]); + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = top_schema.createFromInt( ((int)castBuffer.getShort(0)) & 0xffff ); + //System.out.println("uint 16 "+obj); + break _push; + case CS_UINT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = top_schema.createFromLong( ((long)castBuffer.getInt(0)) & 0xffffffffL ); + //System.out.println("uint 32 "+obj); + break _push; + case CS_UINT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + { + long o = castBuffer.getLong(0); + if(o < 0) { + // FIXME + //obj = GenericBigInteger.valueOf(o & 0x7fffffffL).setBit(31); + throw new UnpackException("uint 64 bigger than 0x7fffffff is not supported"); + } else { + obj = top_schema.createFromLong( o ); + } + } + break _push; + case CS_INT_8: + obj = top_schema.createFromByte( src[n] ); + break _push; + case CS_INT_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = top_schema.createFromShort( castBuffer.getShort(0) ); + break _push; + case CS_INT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = top_schema.createFromInt( castBuffer.getInt(0) ); + break _push; + case CS_INT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + obj = top_schema.createFromLong( castBuffer.getLong(0) ); + break _push; + case CS_RAW_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + trail = ((int)castBuffer.getShort(0)) & 0xffff; + if(trail == 0) { + obj = top_schema.createFromRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + case CS_RAW_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + trail = castBuffer.getInt(0) & 0x7fffffff; + if(trail == 0) { + obj = top_schema.createFromRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + case ACS_RAW_VALUE: + obj = top_schema.createFromRaw(src, n, trail); + break _push; + case CS_ARRAY_16: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IArraySchema)) { + throw new RuntimeException("type error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + obj = new Object[count]; + if(count == 0) { break _push; } // FIXME check IArraySchema + ++top; + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + top_obj = obj; + top_ct = CT_ARRAY_ITEM; + top_count = count; + top_schema = ((IArraySchema)top_schema).getElementSchema(0); + break _header_again; + case CS_ARRAY_32: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IArraySchema)) { + throw new RuntimeException("type error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + obj = new Object[count]; + if(count == 0) { break _push; } // FIXME check IArraySchema + ++top; + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + top_obj = obj; + top_ct = CT_ARRAY_ITEM; + top_count = count; + top_schema = ((IArraySchema)top_schema).getElementSchema(0); + break _header_again; + case CS_MAP_16: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IMapSchema)) { + throw new RuntimeException("type error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + obj = new Object[count*2]; + if(count == 0) { break _push; } // FIXME check IMapSchema + //System.out.println("fixmap count:"+count); + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + ++top; + top_obj = obj; + top_ct = CT_MAP_KEY; + top_count = count; + top_schema = ((IMapSchema)top_schema).getKeySchema(); + break _header_again; + case CS_MAP_32: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IMapSchema)) { + throw new RuntimeException("type error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + obj = new Object[count*2]; + if(count == 0) { break _push; } // FIXME check IMapSchema + //System.out.println("fixmap count:"+count); + ++top; + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + top_obj = obj; + top_ct = CT_MAP_KEY; + top_count = count; + top_schema = ((IMapSchema)top_schema).getKeySchema(); + break _header_again; + default: + throw new UnpackException("parse error"); + } + + } // _fixed_trail_again + } while(true); + } // _push + + do { + _push: { + //System.out.println("push top:"+top); + if(top == -1) { + ++i; + data = obj; + finished = true; + break _out; + } + + switch(top_ct) { + case CT_ARRAY_ITEM: { + //System.out.println("array item "+obj); + Object[] ar = (Object[])top_obj; + ar[ar.length - top_count] = obj; + if(--top_count == 0) { + top_obj = stack_obj[top]; + top_ct = stack_ct[top]; + top_count = stack_count[top]; + top_schema = stack_schema[top]; + obj = ((IArraySchema)top_schema).createFromArray(ar); + stack_obj[top] = null; + stack_schema[top] = null; + --top; + break _push; + } else { + top_schema = ((IArraySchema)stack_schema[top]).getElementSchema(ar.length - top_count); + } + break _header_again; + } + case CT_MAP_KEY: { + //System.out.println("map key:"+top+" "+obj); + Object[] mp = (Object[])top_obj; + mp[mp.length - top_count*2] = obj; + top_ct = CT_MAP_VALUE; + top_schema = ((IMapSchema)stack_schema[top]).getValueSchema(); + break _header_again; + } + case CT_MAP_VALUE: { + //System.out.println("map value:"+top+" "+obj); + Object[] mp = (Object[])top_obj; + mp[mp.length - top_count*2 + 1] = obj; + if(--top_count == 0) { + top_obj = stack_obj[top]; + top_ct = stack_ct[top]; + top_count = stack_count[top]; + top_schema = stack_schema[top]; + obj = ((IMapSchema)top_schema).createFromMap(mp); + stack_obj[top] = null; + stack_schema[top] = null; + --top; + break _push; + } + top_ct = CT_MAP_KEY; + break _header_again; + } + default: + throw new UnpackException("parse error"); + } + } // _push + } while(true); + + } // _header_again + cs = CS_HEADER; + ++i; + } while(i < limit); // _out + + return i; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ArraySchema.java b/java-plan3/src/org/msgpack/schema/ArraySchema.java new file mode 100644 index 0000000..24fa758 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ArraySchema.java @@ -0,0 +1,125 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Set; +import java.util.List; +import java.util.ArrayList; +import java.util.RandomAccess; +import java.io.IOException; +import org.msgpack.*; + +public class ArraySchema extends Schema implements IArraySchema { + private Schema elementSchema; + + public ArraySchema(Schema elementSchema) + { + super("array"); + this.elementSchema = elementSchema; + } + + @Override + public String getFullName() + { + return "List<"+elementSchema.getFullName()+">"; + } + + @Override + public String getExpression() + { + return "(array "+elementSchema.getExpression()+")"; + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException + { + if(obj instanceof List) { + ArrayList d = (ArrayList)obj; + pk.packArray(d.size()); + if(obj instanceof RandomAccess) { + for(int i=0; i < d.size(); ++i) { + elementSchema.pack(pk, d.get(i)); + } + } else { + for(Object e : d) { + elementSchema.pack(pk, e); + } + } + + } else if(obj instanceof Set) { + Set d = (Set)obj; + pk.packArray(d.size()); + for(Object e : d) { + elementSchema.pack(pk, e); + } + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(Object obj) throws MessageTypeException + { + if(obj instanceof List) { + List d = (List)obj; + ArrayList ar = new ArrayList(d.size()); + if(obj instanceof RandomAccess) { + for(int i=0; i < d.size(); ++i) { + ar.add( elementSchema.convert(d.get(i)) ); + } + } else { + for(Object e : d) { + ar.add( elementSchema.convert(e) ); + } + } + return ar; + + } else if(obj instanceof Collection) { + Collection d = (Collection)obj; + ArrayList ar = new ArrayList(d.size()); + for(Object e : (Collection)obj) { + ar.add( elementSchema.convert(e) ); + } + return ar; + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Schema getElementSchema(int index) + { + return elementSchema; + } + + @Override + public Object createFromArray(Object[] obj) + { + return Arrays.asList(obj); + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ByteSchema.java b/java-plan3/src/org/msgpack/schema/ByteSchema.java new file mode 100644 index 0000000..3bc7045 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ByteSchema.java @@ -0,0 +1,89 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class ByteSchema extends Schema { + public ByteSchema() { + super("Byte"); + } + + @Override + public String getExpression() { + return "byte"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packByte( ((Number)obj).byteValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Byte) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).byteValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (byte)v; + } + + @Override + public Object createFromShort(short v) { + return (byte)v; + } + + @Override + public Object createFromInt(int v) { + return (byte)v; + } + + @Override + public Object createFromLong(long v) { + return (byte)v; + } + + @Override + public Object createFromFloat(float v) { + return (byte)v; + } + + @Override + public Object createFromDouble(double v) { + return (byte)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ClassGenerator.java b/java-plan3/src/org/msgpack/schema/ClassGenerator.java new file mode 100644 index 0000000..65213aa --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ClassGenerator.java @@ -0,0 +1,241 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.ArrayList; +import java.util.List; +import java.io.IOException; +import java.io.File; +import java.io.Writer; +import org.msgpack.*; + +public class ClassGenerator { + private ClassSchema schema; + private Writer writer; + private int indent; + + private ClassGenerator(Writer writer) { + this.writer = writer; + this.indent = 0; + } + + public static void write(Schema schema, Writer dest) throws IOException { + if(!(schema instanceof ClassSchema)) { + throw new RuntimeException("schema is not class schema"); + } + ClassSchema cs = (ClassSchema)schema; + new ClassGenerator(dest).run(cs); + } + + private void run(ClassSchema cs) throws IOException { + List subclasses = new ArrayList(); + for(FieldSchema f : cs.getFields()) { + findSubclassSchema(subclasses, f.getSchema()); + } + + for(ClassSchema sub : subclasses) { + sub.setNamespace(cs.getNamespace()); + sub.setImports(cs.getImports()); + } + + this.schema = cs; + + writeHeader(); + + writeClass(); + + for(ClassSchema sub : subclasses) { + this.schema = sub; + writeSubclass(); + } + + writeFooter(); + + this.schema = null; + writer.flush(); + } + + private void findSubclassSchema(List dst, Schema s) { + if(s instanceof ClassSchema) { + ClassSchema cs = (ClassSchema)s; + if(!dst.contains(cs)) { dst.add(cs); } + for(FieldSchema f : cs.getFields()) { + findSubclassSchema(dst, f.getSchema()); + } + } else if(s instanceof ArraySchema) { + ArraySchema as = (ArraySchema)s; + findSubclassSchema(dst, as.getElementSchema(0)); + } else if(s instanceof MapSchema) { + MapSchema as = (MapSchema)s; + findSubclassSchema(dst, as.getKeySchema()); + findSubclassSchema(dst, as.getValueSchema()); + } + } + + private void writeHeader() throws IOException { + if(schema.getNamespace() != null) { + line("package "+schema.getNamespace()+";"); + line(); + } + line("import java.util.*;"); + line("import java.io.*;"); + line("import org.msgpack.*;"); + line("import org.msgpack.schema.ClassSchema;"); + line("import org.msgpack.schema.FieldSchema;"); + } + + private void writeFooter() throws IOException { + line(); + } + + private void writeClass() throws IOException { + line(); + line("public final class "+schema.getName()+" implements MessagePackable, MessageMergeable"); + line("{"); + pushIndent(); + writeSchema(); + writeMemberVariables(); + writeMemberFunctions(); + popIndent(); + line("}"); + } + + private void writeSubclass() throws IOException { + line(); + line("final class "+schema.getName()+" implements MessagePackable, MessageMergeable"); + line("{"); + pushIndent(); + writeSchema(); + writeMemberVariables(); + writeMemberFunctions(); + popIndent(); + line("}"); + } + + private void writeSchema() throws IOException { + line("private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load(\""+schema.getExpression()+"\");"); + line("public static ClassSchema getSchema() { return _SCHEMA; }"); + } + + private void writeMemberVariables() throws IOException { + line(); + for(FieldSchema f : schema.getFields()) { + line("public "+f.getSchema().getFullName()+" "+f.getName()+";"); + } + } + + private void writeMemberFunctions() throws IOException { + // void messagePack(Packer pk) + // boolean equals(Object obj) + // int hashCode() + // void set(int _index, Object _value) + // Object get(int _index); + // getXxx() + // setXxx(Xxx xxx) + writeConstructors(); + writeAccessors(); + writePackFunction(); + writeMergeFunction(); + writeFactoryFunction(); + } + + private void writeConstructors() throws IOException { + line(); + line("public "+schema.getName()+"() { }"); + } + + private void writeAccessors() throws IOException { + // FIXME + //line(); + //for(FieldSchema f : schema.getFields()) { + // line(""); + //} + } + + private void writePackFunction() throws IOException { + line(); + line("@Override"); + line("public void messagePack(Packer _pk) throws IOException"); + line("{"); + pushIndent(); + line("_pk.packArray("+schema.getFields().length+");"); + line("FieldSchema[] _fields = _SCHEMA.getFields();"); + int i = 0; + for(FieldSchema f : schema.getFields()) { + line("_fields["+i+"].getSchema().pack(_pk, "+f.getName()+");"); + ++i; + } + popIndent(); + line("}"); + } + + private void writeMergeFunction() throws IOException { + line(); + line("@Override"); + line("@SuppressWarnings(\"unchecked\")"); + line("public void messageMerge(Object obj) throws MessageTypeException"); + line("{"); + pushIndent(); + line("Object[] _source = ((List)obj).toArray();"); + line("FieldSchema[] _fields = _SCHEMA.getFields();"); + int i = 0; + for(FieldSchema f : schema.getFields()) { + line("if(_source.length <= "+i+") { return; } this."+f.getName()+" = ("+f.getSchema().getFullName()+")_fields["+i+"].getSchema().convert(_source["+i+"]);"); + ++i; + } + popIndent(); + line("}"); + } + + private void writeFactoryFunction() throws IOException { + line(); + line("@SuppressWarnings(\"unchecked\")"); + line("public static "+schema.getName()+" createFromMessage(Object[] _message)"); + line("{"); + pushIndent(); + line(schema.getName()+" _self = new "+schema.getName()+"();"); + int i = 0; + for(FieldSchema f : schema.getFields()) { + line("if(_message.length <= "+i+") { return _self; } _self."+f.getName()+" = ("+f.getSchema().getFullName()+")_message["+i+"];"); + ++i; + } + line("return _self;"); + popIndent(); + line("}"); + } + + private void line(String str) throws IOException { + for(int i=0; i < indent; ++i) { + writer.write("\t"); + } + writer.write(str+"\n"); + } + + private void line() throws IOException { + writer.write("\n"); + } + + private void pushIndent() { + indent += 1; + } + + private void popIndent() { + indent -= 1; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ClassSchema.java b/java-plan3/src/org/msgpack/schema/ClassSchema.java new file mode 100644 index 0000000..75315e7 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ClassSchema.java @@ -0,0 +1,95 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Arrays; +import java.util.List; +import org.msgpack.*; + +public abstract class ClassSchema extends Schema implements IArraySchema { + protected FieldSchema[] fields; + protected List imports; + protected String namespace; + protected String fqdn; + + public ClassSchema( + String name, String namespace, + List imports, List fields) { + super(name); + this.namespace = namespace; + this.imports = imports; // FIXME clone? + this.fields = new FieldSchema[fields.size()]; + System.arraycopy(fields.toArray(), 0, this.fields, 0, fields.size()); + if(namespace == null) { + this.fqdn = name; + } else { + this.fqdn = namespace+"."+name; + } + } + + public final FieldSchema[] getFields() { + return fields; + } + + String getNamespace() { + return namespace; + } + + List getImports() { + return imports; + } + + void setNamespace(String namespace) { + this.namespace = namespace; + } + + void setImports(List imports) { + this.imports = imports; // FIXME clone? + } + + //@Override + //public String getFullName() + //{ + // if(namespace == null) { + // return getName(); + // } else { + // return namespace+"."+getName(); + // } + //} + + @Override + public String getExpression() { + StringBuffer b = new StringBuffer(); + b.append("(class "); + b.append(getName()); + if(namespace != null) { + b.append(" (package "+namespace+")"); + } + for(FieldSchema f : fields) { + b.append(" "+f.getExpression()); + } + b.append(")"); + return b.toString(); + } + + public boolean equals(SpecificClassSchema o) { + return (namespace != null ? namespace.equals(o.getNamespace()) : o.getNamespace() == null) && + getName().equals(o.getName()); + } +} + diff --git a/java-plan3/src/org/msgpack/schema/DoubleSchema.java b/java-plan3/src/org/msgpack/schema/DoubleSchema.java new file mode 100644 index 0000000..feffbb9 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/DoubleSchema.java @@ -0,0 +1,84 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class DoubleSchema extends Schema { + public DoubleSchema() { + super("Double"); + } + + @Override + public String getExpression() { + return "double"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packDouble( ((Number)obj).doubleValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Double) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).doubleValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (double)v; + } + + @Override + public Object createFromShort(short v) { + return (double)v; + } + + @Override + public Object createFromInt(int v) { + return (double)v; + } + + @Override + public Object createFromFloat(float v) { + return (double)v; + } + + @Override + public Object createFromDouble(double v) { + return (double)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/FieldSchema.java b/java-plan3/src/org/msgpack/schema/FieldSchema.java new file mode 100644 index 0000000..3391f2b --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/FieldSchema.java @@ -0,0 +1,43 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import org.msgpack.Schema; + +public class FieldSchema { + private String name; + private Schema schema; + + public FieldSchema(String name, Schema schema) { + this.name = name; + this.schema = schema; + } + + public final String getName() { + return name; + } + + public final Schema getSchema() { + return schema; + } + + public String getExpression() { + return "(field "+name+" "+schema.getExpression()+")"; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/FloatSchema.java b/java-plan3/src/org/msgpack/schema/FloatSchema.java new file mode 100644 index 0000000..2f4240a --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/FloatSchema.java @@ -0,0 +1,84 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class FloatSchema extends Schema { + public FloatSchema() { + super("Float"); + } + + @Override + public String getExpression() { + return "float"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packFloat( ((Number)obj).floatValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Float) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).floatValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (float)v; + } + + @Override + public Object createFromShort(short v) { + return (float)v; + } + + @Override + public Object createFromInt(int v) { + return (float)v; + } + + @Override + public Object createFromFloat(float v) { + return (float)v; + } + + @Override + public Object createFromDouble(double v) { + return (float)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/GenericClassSchema.java b/java-plan3/src/org/msgpack/schema/GenericClassSchema.java new file mode 100644 index 0000000..736bdfa --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/GenericClassSchema.java @@ -0,0 +1,91 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.io.IOException; +import org.msgpack.*; + +public class GenericClassSchema extends ClassSchema { + public GenericClassSchema( + String name, String namespace, + List imports, List fields) { + super(name, namespace, imports, fields); + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Map) { + Map d = (Map)obj; + pk.packArray(fields.length); + for(int i=0; i < fields.length; ++i) { + FieldSchema f = fields[i]; + f.getSchema().pack(pk, d.get(f.getName())); + } + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Collection) { + // FIXME optimize + return createFromArray( ((Collection)obj).toArray() ); + + } else if(obj instanceof Map) { + HashMap m = new HashMap(fields.length); + Map d = (Map)obj; + for(int i=0; i < fields.length; ++i) { + FieldSchema f = fields[i]; + String fieldName = f.getName(); + m.put(fieldName, f.getSchema().convert(d.get(fieldName))); + } + return m; + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + public Schema getElementSchema(int index) { + // FIXME check index < fields.length + return fields[index].getSchema(); + } + + public Object createFromArray(Object[] obj) { + HashMap m = new HashMap(fields.length); + int i=0; + for(; i < obj.length; ++i) { + m.put(fields[i].getName(), obj[i]); + } + for(; i < fields.length; ++i) { + m.put(fields[i].getName(), null); + } + return m; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/GenericSchema.java b/java-plan3/src/org/msgpack/schema/GenericSchema.java new file mode 100644 index 0000000..52e0161 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/GenericSchema.java @@ -0,0 +1,192 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Arrays; +import java.util.List; +import java.util.HashMap; +import java.io.IOException; +import org.msgpack.*; +//import org.msgpack.generic.*; + +public class GenericSchema extends Schema implements IArraySchema, IMapSchema { + public GenericSchema() { + super("Object"); + } + + @Override + public String getExpression() { + return "object"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + pk.pack(obj); + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + return obj; + } + + @Override + public Schema getElementSchema(int index) { + return this; + } + + @Override + public Schema getKeySchema() { + return this; + } + + @Override + public Schema getValueSchema() { + return this; + } + + @Override + public Object createFromNil() { + return null; + } + + @Override + public Object createFromBoolean(boolean v) { + return v; + } + + @Override + public Object createFromByte(byte v) { + return v; + } + + @Override + public Object createFromShort(short v) { + return v; + } + + @Override + public Object createFromInt(int v) { + return v; + } + + @Override + public Object createFromLong(long v) { + return v; + } + + @Override + public Object createFromFloat(float v) { + return v; + } + + @Override + public Object createFromDouble(double v) { + return v; + } + + @Override + public Object createFromRaw(byte[] b, int offset, int length) { + byte[] bytes = new byte[length]; + System.arraycopy(b, offset, bytes, 0, length); + return bytes; + } + + @Override + public Object createFromArray(Object[] obj) { + return Arrays.asList(obj); + } + + @Override + @SuppressWarnings("unchecked") + public Object createFromMap(Object[] obj) { + HashMap m = new HashMap(obj.length / 2); + int i = 0; + while(i < obj.length) { + Object k = obj[i++]; + Object v = obj[i++]; + m.put(k, v); + } + return m; + } + + /* + @Override + public Object createFromNil() { + return null; + } + + @Override + public Object createFromBoolean(boolean v) { + return new GenericBoolean(v); + } + + @Override + public Object createFromFromByte(byte v) { + return new GenericByte(v); + } + + @Override + public Object createFromShort(short v) { + return new GenericShort(v); + } + + @Override + public Object createFromInt(int v) { + return new GenericInt(v); + } + + @Override + public Object createFromLong(long v) { + return new GenericLong(v); + } + + @Override + public Object createFromFloat(float v) { + return new GenericFloat(v); + } + + @Override + public Object createFromDouble(double v) { + return new GenericDouble(v); + } + + @Override + public Object createFromRaw(byte[] b, int offset, int length) { + return new GenericRaw(b, offset, length); + } + + @Override + public Object createFromArray(Object[] obj) { + // FIXME GenericArray + return Arrays.asList(obj); + } + + @Override + public Object createFromMap(Object[] obj) { + GenericMap m = new GenericMap(obj.length / 2); + int i = 0; + while(i < obj.length) { + Object k = obj[i++]; + Object v = obj[i++]; + m.put(k, v); + } + return m; + } + */ +} + diff --git a/java-plan3/src/org/msgpack/schema/IArraySchema.java b/java-plan3/src/org/msgpack/schema/IArraySchema.java new file mode 100644 index 0000000..ccc6d14 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/IArraySchema.java @@ -0,0 +1,26 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import org.msgpack.Schema; + +public interface IArraySchema { + public Schema getElementSchema(int index); + public Object createFromArray(Object[] obj); +} + diff --git a/java-plan3/src/org/msgpack/schema/IMapSchema.java b/java-plan3/src/org/msgpack/schema/IMapSchema.java new file mode 100644 index 0000000..60b3e8e --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/IMapSchema.java @@ -0,0 +1,27 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import org.msgpack.Schema; + +public interface IMapSchema { + public Schema getKeySchema(); + public Schema getValueSchema(); + public Object createFromMap(Object[] obj); +} + diff --git a/java-plan3/src/org/msgpack/schema/IntSchema.java b/java-plan3/src/org/msgpack/schema/IntSchema.java new file mode 100644 index 0000000..c54c0ae --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/IntSchema.java @@ -0,0 +1,89 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class IntSchema extends Schema { + public IntSchema() { + super("Integer"); + } + + @Override + public String getExpression() { + return "int"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packInt( ((Number)obj).intValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Integer) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).intValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (int)v; + } + + @Override + public Object createFromShort(short v) { + return (int)v; + } + + @Override + public Object createFromInt(int v) { + return (int)v; + } + + @Override + public Object createFromLong(long v) { + return (int)v; + } + + @Override + public Object createFromFloat(float v) { + return (int)v; + } + + @Override + public Object createFromDouble(double v) { + return (int)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/LongSchema.java b/java-plan3/src/org/msgpack/schema/LongSchema.java new file mode 100644 index 0000000..ccf3043 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/LongSchema.java @@ -0,0 +1,89 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class LongSchema extends Schema { + public LongSchema() { + super("Long"); + } + + @Override + public String getExpression() { + return "long"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packLong( ((Number)obj).longValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Long) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).longValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (long)v; + } + + @Override + public Object createFromShort(short v) { + return (long)v; + } + + @Override + public Object createFromInt(int v) { + return (long)v; + } + + @Override + public Object createFromLong(long v) { + return (long)v; + } + + @Override + public Object createFromFloat(float v) { + return (long)v; + } + + @Override + public Object createFromDouble(double v) { + return (long)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/MapSchema.java b/java-plan3/src/org/msgpack/schema/MapSchema.java new file mode 100644 index 0000000..71629b0 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/MapSchema.java @@ -0,0 +1,102 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Map; +import java.util.HashMap; +import java.io.IOException; +import org.msgpack.*; + +public class MapSchema extends Schema implements IMapSchema { + private Schema keySchema; + private Schema valueSchema; + + public MapSchema(Schema keySchema, Schema valueSchema) { + super("map"); + this.keySchema = keySchema; + this.valueSchema = valueSchema; + } + + @Override + public String getFullName() { + return "HashList<"+keySchema.getFullName()+", "+valueSchema.getFullName()+">"; + } + + @Override + public String getExpression() { + return "(map "+keySchema.getExpression()+" "+valueSchema.getExpression()+")"; + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Map) { + Map d = (Map)obj; + pk.packMap(d.size()); + for(Map.Entry e : d.entrySet()) { + keySchema.pack(pk, e.getKey()); + valueSchema.pack(pk, e.getValue()); + } + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Map) { + Map d = (Map)obj; + Map m = new HashMap(); + for(Map.Entry e : d.entrySet()) { + m.put(keySchema.convert(e.getKey()), valueSchema.convert(e.getValue())); + } + return m; + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Schema getKeySchema() { + return keySchema; + } + + @Override + public Schema getValueSchema() { + return valueSchema; + } + + @Override + public Object createFromMap(Object[] obj) { + HashMap m = new HashMap(obj.length / 2); + int i = 0; + while(i < obj.length) { + Object k = obj[i++]; + Object v = obj[i++]; + m.put(k, v); + } + return m; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/RawSchema.java b/java-plan3/src/org/msgpack/schema/RawSchema.java new file mode 100644 index 0000000..582f766 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/RawSchema.java @@ -0,0 +1,105 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.nio.ByteBuffer; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import org.msgpack.*; + +public class RawSchema extends Schema { + public RawSchema() { + super("raw"); + } + + public String getFullName() { + return "byte[]"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + // FIXME instanceof GenericObject + if(obj instanceof byte[]) { + byte[] d = (byte[])obj; + pk.packRaw(d.length); + pk.packRawBody(d); + + } else if(obj instanceof ByteBuffer) { + ByteBuffer d = (ByteBuffer)obj; + if(!d.hasArray()) { + throw MessageTypeException.invalidConvert(obj, this); + } + pk.packRaw(d.capacity()); + pk.packRawBody(d.array(), d.position(), d.capacity()); + + } else if(obj instanceof String) { + try { + byte[] d = ((String)obj).getBytes("UTF-8"); + pk.packRaw(d.length); + pk.packRawBody(d); + } catch (UnsupportedEncodingException e) { + throw MessageTypeException.invalidConvert(obj, this); + } + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + // FIXME instanceof GenericObject + if(obj instanceof byte[]) { + // FIXME copy? + //byte[] d = (byte[])obj; + //byte[] v = new byte[d.length]; + //System.arraycopy(d, 0, v, 0, d.length); + //return v; + return obj; + + } else if(obj instanceof ByteBuffer) { + ByteBuffer d = (ByteBuffer)obj; + byte[] v = new byte[d.capacity()]; + int pos = d.position(); + d.get(v); + d.position(pos); + return v; + + } else if(obj instanceof String) { + try { + return ((String)obj).getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw MessageTypeException.invalidConvert(obj, this); + } + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromRaw(byte[] b, int offset, int length) { + byte[] d = new byte[length]; + System.arraycopy(b, offset, d, 0, length); + return d; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java b/java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java new file mode 100644 index 0000000..fb94adf --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java @@ -0,0 +1,64 @@ +package org.msgpack.schema; + +import java.util.Arrays; +import java.util.List; +import java.lang.reflect.*; +import org.msgpack.*; + +// FIXME +public abstract class ReflectionClassSchema extends ClassSchema { + private Constructor constructorCache; + + public ReflectionClassSchema(String name, List fields, String namespace, List imports) { + super(name, namespace, imports, fields); + } + + /* + Schema getElementSchema(int index) + { + // FIXME check index < fields.length + fields[index].getSchema(); + } + + Object createFromArray(Object[] obj) + { + Object o = newInstance(); + ((MessageConvertable)o).messageConvert(obj); + return o; + } + + Object newInstance() + { + if(constructorCache == null) { + cacheConstructor(); + } + try { + return constructorCache.newInstance((Object[])null); + } catch (InvocationTargetException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (InstantiationException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (IllegalAccessException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } + } + + private void cacheConstructor() + { + try { + Class c = Class.forName(fqdn); + int index = 0; + for(SpecificFieldSchema f : fields) { + f.cacheField(c, index++); + } + constructorCache = c.getDeclaredConstructor((Class[])null); + constructorCache.setAccessible(true); + } catch(ClassNotFoundException e) { + throw new RuntimeException("class not found: "+fqdn); + } catch (NoSuchMethodException e) { + throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); + } + } + */ +} + diff --git a/java-plan3/src/org/msgpack/schema/SSchemaParser.java b/java-plan3/src/org/msgpack/schema/SSchemaParser.java new file mode 100644 index 0000000..c6bbc77 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/SSchemaParser.java @@ -0,0 +1,254 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Stack; +import java.util.regex.Pattern; +import java.util.regex.Matcher; +import org.msgpack.*; + +// FIXME exception class + +public class SSchemaParser { + public static Schema parse(String source) { + return new SSchemaParser(false).run(source); + } + + public static Schema load(String source) { + return new SSchemaParser(true).run(source); + } + + private static abstract class SExp { + boolean isAtom() { return false; } + public String getAtom() { return null; } + + boolean isTuple() { return false; } + public SExp getTuple(int i) { return null; } + public int size() { return 0; } + public boolean empty() { return size() == 0; } + Iterator iterator(int offset) { return null; } + } + + private static class SAtom extends SExp { + private String atom; + + SAtom(String atom) { this.atom = atom; } + + boolean isAtom() { return true; } + public String getAtom() { return atom; } + + public String toString() { return atom; } + } + + private static class STuple extends SExp { + private List tuple; + + STuple() { this.tuple = new ArrayList(); } + + public void add(SExp e) { tuple.add(e); } + + boolean isTuple() { return true; } + public SExp getTuple(int i) { return tuple.get(i); } + public int size() { return tuple.size(); } + + Iterator iterator(int skip) { + Iterator i = tuple.iterator(); + for(int s=0; s < skip; ++s) { i.next(); } + return i; + } + + public String toString() { + if(tuple.isEmpty()) { return "()"; } + Iterator i = tuple.iterator(); + StringBuffer o = new StringBuffer(); + o.append("(").append(i.next()); + while(i.hasNext()) { o.append(" ").append(i.next()); } + o.append(")"); + return o.toString(); + } + } + + boolean specificClass; + + private SSchemaParser(boolean specificClass) { + this.specificClass = specificClass; + } + + private static Pattern pattern = Pattern.compile( + "(?:\\s+)|([\\(\\)]|[\\d\\w\\.]+)"); + + private Schema run(String source) { + Matcher m = pattern.matcher(source); + + Stack stack = new Stack(); + String token; + + while(true) { + while(true) { + if(!m.find()) { throw new RuntimeException("unexpected end of file"); } + token = m.group(1); + if(token != null) { break; } + } + + if(token.equals("(")) { + stack.push(new STuple()); + } else if(token.equals(")")) { + STuple top = stack.pop(); + if(stack.empty()) { + stack.push(top); + break; + } + stack.peek().add(top); + } else { + if(stack.empty()) { + throw new RuntimeException("unexpected token '"+token+"'"); + } + stack.peek().add(new SAtom(token)); + } + } + + while(true) { + if(!m.find()) { break; } + token = m.group(1); + if(token != null) { throw new RuntimeException("unexpected token '"+token+"'"); } + } + + return readType( stack.pop() ); + } + + private Schema readType(SExp exp) { + if(exp.isAtom()) { + String type = exp.getAtom(); + if(type.equals("string")) { + return new StringSchema(); + } else if(type.equals("raw")) { + return new RawSchema(); + } else if(type.equals("byte")) { + return new ByteSchema(); + } else if(type.equals("short")) { + return new ShortSchema(); + } else if(type.equals("int")) { + return new IntSchema(); + } else if(type.equals("long")) { + return new LongSchema(); + } else if(type.equals("float")) { + return new FloatSchema(); + } else if(type.equals("double")) { + return new DoubleSchema(); + } else if(type.equals("object")) { + return new GenericSchema(); + } else { + throw new RuntimeException("byte, short, int, long, float, double, raw, string or object is expected but got '"+type+"': "+exp); + } + } else { + String type = exp.getTuple(0).getAtom(); + if(type.equals("class")) { + return parseClass(exp); + } else if(type.equals("array")) { + return parseArray(exp); + } else if(type.equals("map")) { + return parseMap(exp); + } else { + throw new RuntimeException("class, array or map is expected but got '"+type+"': "+exp); + } + } + } + + private ClassSchema parseClass(SExp exp) { + if(exp.size() < 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("class is (class NAME CLASS_BODY): "+exp); + } + + String namespace = null; + List imports = new ArrayList(); + String name = exp.getTuple(1).getAtom(); + List fields = new ArrayList(); + + for(Iterator i=exp.iterator(2); i.hasNext();) { + SExp subexp = i.next(); + if(!subexp.isTuple() || subexp.empty() || !subexp.getTuple(0).isAtom()) { + throw new RuntimeException("field, package or import is expected: "+subexp); + } + String type = subexp.getTuple(0).getAtom(); + if(type.equals("field")) { + fields.add( parseField(subexp) ); + } else if(type.equals("package")) { + if(namespace != null) { + throw new RuntimeException("duplicated package definition: "+subexp); + } + namespace = parseNamespace(subexp); + } else if(type.equals("import")) { + imports.add( parseImport(subexp) ); + } else { + throw new RuntimeException("field, package or import is expected but got '"+type+"': "+subexp); + } + } + + if(specificClass) { + return new SpecificClassSchema(name, namespace, imports, fields); + } else { + return new GenericClassSchema(name, namespace, imports, fields); + } + } + + private ArraySchema parseArray(SExp exp) { + if(exp.size() != 2) { + throw new RuntimeException("array is (array ELEMENT_TYPE): "+exp); + } + Schema elementType = readType(exp.getTuple(1)); + return new ArraySchema(elementType); + } + + private MapSchema parseMap(SExp exp) { + if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("map is (map KEY_TYPE VALUE_TYPE): "+exp); + } + Schema keyType = readType(exp.getTuple(1)); + Schema valueType = readType(exp.getTuple(2)); + return new MapSchema(keyType, valueType); + } + + private String parseNamespace(SExp exp) { + if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("package is (package NAME): "+exp); + } + String name = exp.getTuple(1).getAtom(); + return name; + } + + private String parseImport(SExp exp) { + if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("import is (import NAME): "+exp); + } + String name = exp.getTuple(1).getAtom(); + return name; + } + + private FieldSchema parseField(SExp exp) { + if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("field is (field NAME TYPE): "+exp); + } + String name = exp.getTuple(1).getAtom(); + Schema type = readType(exp.getTuple(2)); + return new FieldSchema(name, type); + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ShortSchema.java b/java-plan3/src/org/msgpack/schema/ShortSchema.java new file mode 100644 index 0000000..089a024 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ShortSchema.java @@ -0,0 +1,89 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class ShortSchema extends Schema { + public ShortSchema() { + super("Short"); + } + + @Override + public String getExpression() { + return "short"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packShort( ((Number)obj).shortValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Short) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).shortValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (short)v; + } + + @Override + public Object createFromShort(short v) { + return (short)v; + } + + @Override + public Object createFromInt(int v) { + return (short)v; + } + + @Override + public Object createFromLong(long v) { + return (short)v; + } + + @Override + public Object createFromFloat(float v) { + return (short)v; + } + + @Override + public Object createFromDouble(double v) { + return (short)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/SpecificClassSchema.java b/java-plan3/src/org/msgpack/schema/SpecificClassSchema.java new file mode 100644 index 0000000..81e5e00 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/SpecificClassSchema.java @@ -0,0 +1,122 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Collection; +import java.util.List; +import java.lang.reflect.*; +import java.io.IOException; +import org.msgpack.*; + +public class SpecificClassSchema extends ClassSchema { + private Class classCache; + private Method factoryCache; + private Constructor constructorCache; + + public SpecificClassSchema( + String name, String namespace, + List imports, List fields) { + super(name, namespace, imports, fields); + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException { + if(obj == null) { + pk.packNil(); + return; + } + if(classCache == null) { + cacheFactory(); + } + if(classCache.isInstance(obj)) { + ((MessagePackable)obj).messagePack(pk); + } else { + // FIXME Map + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Collection) { + if(constructorCache == null) { + cacheConstructor(); + } + try { + MessageMergeable o = (MessageMergeable)constructorCache.newInstance((Object[])null); + o.messageMerge(obj); + return o; + } catch (InvocationTargetException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (InstantiationException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (IllegalAccessException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + public Schema getElementSchema(int index) { + // FIXME check index < fields.length + return fields[index].getSchema(); + } + + public Object createFromArray(Object[] obj) { + if(factoryCache == null) { + cacheFactory(); + } + try { + return factoryCache.invoke(null, new Object[]{obj}); + } catch (InvocationTargetException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getCause()); + } catch (IllegalAccessException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } + } + + @SuppressWarnings("unchecked") + private void cacheFactory() { + try { + classCache = Class.forName(fqdn); + factoryCache = classCache.getDeclaredMethod("createFromMessage", new Class[]{Object[].class}); + factoryCache.setAccessible(true); + } catch(ClassNotFoundException e) { + throw new RuntimeException("class not found: "+fqdn); + } catch (NoSuchMethodException e) { + throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); + } + } + + @SuppressWarnings("unchecked") + private void cacheConstructor() { + try { + classCache = Class.forName(fqdn); + constructorCache = classCache.getDeclaredConstructor((Class[])null); + constructorCache.setAccessible(true); + } catch(ClassNotFoundException e) { + throw new RuntimeException("class not found: "+fqdn); + } catch (NoSuchMethodException e) { + throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); + } + } +} + diff --git a/java-plan3/src/org/msgpack/schema/StringSchema.java b/java-plan3/src/org/msgpack/schema/StringSchema.java new file mode 100644 index 0000000..f5e0bf1 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/StringSchema.java @@ -0,0 +1,111 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.nio.ByteBuffer; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import org.msgpack.*; + +public class StringSchema extends Schema { + public StringSchema() { + super("string"); + } + + @Override + public String getFullName() { + return "String"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + // FIXME instanceof GenericObject + if(obj instanceof String) { + try { + byte[] d = ((String)obj).getBytes("UTF-8"); + pk.packRaw(d.length); + pk.packRawBody(d); + } catch (UnsupportedEncodingException e) { + throw MessageTypeException.invalidConvert(obj, this); + } + + } else if(obj instanceof byte[]) { + byte[] d = (byte[])obj; + pk.packRaw(d.length); + pk.packRawBody(d); + + } else if(obj instanceof ByteBuffer) { + ByteBuffer d = (ByteBuffer)obj; + if(!d.hasArray()) { + throw MessageTypeException.invalidConvert(obj, this); + } + pk.packRaw(d.capacity()); + pk.packRawBody(d.array(), d.position(), d.capacity()); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + // FIXME instanceof GenericObject + if(obj instanceof String) { + return obj; + + } else if(obj instanceof byte[]) { + try { + return new String((byte[])obj, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw MessageTypeException.invalidConvert(obj, this); + } + + } else if(obj instanceof ByteBuffer) { + ByteBuffer d = (ByteBuffer)obj; + try { + if(d.hasArray()) { + return new String(d.array(), d.position(), d.capacity(), "UTF-8"); + } else { + byte[] v = new byte[d.capacity()]; + int pos = d.position(); + d.get(v); + d.position(pos); + return new String(v, "UTF-8"); + } + } catch (UnsupportedEncodingException e) { + throw MessageTypeException.invalidConvert(obj, this); + } + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromRaw(byte[] b, int offset, int length) { + try { + return new String(b, offset, length, "UTF-8"); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } +} + diff --git a/java-plan3/test/Generate.java b/java-plan3/test/Generate.java new file mode 100644 index 0000000..1b72e90 --- /dev/null +++ b/java-plan3/test/Generate.java @@ -0,0 +1,38 @@ +import java.io.*; +import java.util.*; +import org.msgpack.*; +import org.msgpack.schema.*; + +public class Generate { + public static void main(String[] args) throws IOException + { + String source = + "(class MediaContent"+ + " (package serializers.msgpack)"+ + " (field image (array (class Image"+ + " (field uri string)"+ + " (field title string)"+ + " (field width int)"+ + " (field height int)"+ + " (field size int))))"+ + " (field media (class Media"+ + " (field uri string)"+ + " (field title string)"+ + " (field width int)"+ + " (field height int)"+ + " (field format string)"+ + " (field duration long)"+ + " (field size long)"+ + " (field bitrate int)"+ + " (field person (array string))"+ + " (field player int)"+ + " (field copyright string)))"+ + " )"; + + Schema schema = Schema.parse(source); + + Writer output = new OutputStreamWriter(System.out); + ClassGenerator.write(schema, output); + } +} + diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java new file mode 100644 index 0000000..5dfbc8d --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java @@ -0,0 +1,173 @@ +package serializers.msgpack; + +import java.util.*; +import java.io.*; +import org.msgpack.*; +import org.msgpack.schema.ClassSchema; +import org.msgpack.schema.FieldSchema; + +public final class MediaContent implements MessagePackable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class MediaContent (package serializers.msgpack) (field image (array (class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public List image; + public Media media; + + public MediaContent() { } + + @Override + public void messagePack(Packer _pk) throws IOException + { + _pk.packArray(2); + FieldSchema[] _fields = _SCHEMA.getFields(); + _fields[0].getSchema().pack(_pk, image); + _fields[1].getSchema().pack(_pk, media); + } + + @Override + @SuppressWarnings("unchecked") + public void messageMerge(Object obj) throws MessageTypeException + { + Object[] _source = ((List)obj).toArray(); + FieldSchema[] _fields = _SCHEMA.getFields(); + if(_source.length <= 0) { return; } this.image = (List)_fields[0].getSchema().convert(_source[0]); + if(_source.length <= 1) { return; } this.media = (Media)_fields[1].getSchema().convert(_source[1]); + } + + @SuppressWarnings("unchecked") + public static MediaContent createFromMessage(Object[] _message) + { + MediaContent _self = new MediaContent(); + if(_message.length <= 0) { return _self; } _self.image = (List)_message[0]; + if(_message.length <= 1) { return _self; } _self.media = (Media)_message[1]; + return _self; + } +} + +final class Image implements MessagePackable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public String uri; + public String title; + public Integer width; + public Integer height; + public Integer size; + + public Image() { } + + @Override + public void messagePack(Packer _pk) throws IOException + { + _pk.packArray(5); + FieldSchema[] _fields = _SCHEMA.getFields(); + _fields[0].getSchema().pack(_pk, uri); + _fields[1].getSchema().pack(_pk, title); + _fields[2].getSchema().pack(_pk, width); + _fields[3].getSchema().pack(_pk, height); + _fields[4].getSchema().pack(_pk, size); + } + + @Override + @SuppressWarnings("unchecked") + public void messageMerge(Object obj) throws MessageTypeException + { + Object[] _source = ((List)obj).toArray(); + FieldSchema[] _fields = _SCHEMA.getFields(); + if(_source.length <= 0) { return; } this.uri = (String)_fields[0].getSchema().convert(_source[0]); + if(_source.length <= 1) { return; } this.title = (String)_fields[1].getSchema().convert(_source[1]); + if(_source.length <= 2) { return; } this.width = (Integer)_fields[2].getSchema().convert(_source[2]); + if(_source.length <= 3) { return; } this.height = (Integer)_fields[3].getSchema().convert(_source[3]); + if(_source.length <= 4) { return; } this.size = (Integer)_fields[4].getSchema().convert(_source[4]); + } + + @SuppressWarnings("unchecked") + public static Image createFromMessage(Object[] _message) + { + Image _self = new Image(); + if(_message.length <= 0) { return _self; } _self.uri = (String)_message[0]; + if(_message.length <= 1) { return _self; } _self.title = (String)_message[1]; + if(_message.length <= 2) { return _self; } _self.width = (Integer)_message[2]; + if(_message.length <= 3) { return _self; } _self.height = (Integer)_message[3]; + if(_message.length <= 4) { return _self; } _self.size = (Integer)_message[4]; + return _self; + } +} + +final class Media implements MessagePackable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public String uri; + public String title; + public Integer width; + public Integer height; + public String format; + public Long duration; + public Long size; + public Integer bitrate; + public List person; + public Integer player; + public String copyright; + + public Media() { } + + @Override + public void messagePack(Packer _pk) throws IOException + { + _pk.packArray(11); + FieldSchema[] _fields = _SCHEMA.getFields(); + _fields[0].getSchema().pack(_pk, uri); + _fields[1].getSchema().pack(_pk, title); + _fields[2].getSchema().pack(_pk, width); + _fields[3].getSchema().pack(_pk, height); + _fields[4].getSchema().pack(_pk, format); + _fields[5].getSchema().pack(_pk, duration); + _fields[6].getSchema().pack(_pk, size); + _fields[7].getSchema().pack(_pk, bitrate); + _fields[8].getSchema().pack(_pk, person); + _fields[9].getSchema().pack(_pk, player); + _fields[10].getSchema().pack(_pk, copyright); + } + + @Override + @SuppressWarnings("unchecked") + public void messageMerge(Object obj) throws MessageTypeException + { + Object[] _source = ((List)obj).toArray(); + FieldSchema[] _fields = _SCHEMA.getFields(); + if(_source.length <= 0) { return; } this.uri = (String)_fields[0].getSchema().convert(_source[0]); + if(_source.length <= 1) { return; } this.title = (String)_fields[1].getSchema().convert(_source[1]); + if(_source.length <= 2) { return; } this.width = (Integer)_fields[2].getSchema().convert(_source[2]); + if(_source.length <= 3) { return; } this.height = (Integer)_fields[3].getSchema().convert(_source[3]); + if(_source.length <= 4) { return; } this.format = (String)_fields[4].getSchema().convert(_source[4]); + if(_source.length <= 5) { return; } this.duration = (Long)_fields[5].getSchema().convert(_source[5]); + if(_source.length <= 6) { return; } this.size = (Long)_fields[6].getSchema().convert(_source[6]); + if(_source.length <= 7) { return; } this.bitrate = (Integer)_fields[7].getSchema().convert(_source[7]); + if(_source.length <= 8) { return; } this.person = (List)_fields[8].getSchema().convert(_source[8]); + if(_source.length <= 9) { return; } this.player = (Integer)_fields[9].getSchema().convert(_source[9]); + if(_source.length <= 10) { return; } this.copyright = (String)_fields[10].getSchema().convert(_source[10]); + } + + @SuppressWarnings("unchecked") + public static Media createFromMessage(Object[] _message) + { + Media _self = new Media(); + if(_message.length <= 0) { return _self; } _self.uri = (String)_message[0]; + if(_message.length <= 1) { return _self; } _self.title = (String)_message[1]; + if(_message.length <= 2) { return _self; } _self.width = (Integer)_message[2]; + if(_message.length <= 3) { return _self; } _self.height = (Integer)_message[3]; + if(_message.length <= 4) { return _self; } _self.format = (String)_message[4]; + if(_message.length <= 5) { return _self; } _self.duration = (Long)_message[5]; + if(_message.length <= 6) { return _self; } _self.size = (Long)_message[6]; + if(_message.length <= 7) { return _self; } _self.bitrate = (Integer)_message[7]; + if(_message.length <= 8) { return _self; } _self.person = (List)_message[8]; + if(_message.length <= 9) { return _self; } _self.player = (Integer)_message[9]; + if(_message.length <= 10) { return _self; } _self.copyright = (String)_message[10]; + return _self; + } +} + diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs new file mode 100644 index 0000000..547ba48 --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs @@ -0,0 +1,21 @@ +(class MediaContent + (package serializers.msgpack) + (field image (array (class Image + (field uri string) + (field title string) + (field width int) + (field height int) + (field size int)))) + (field media (class Media + (field uri string) + (field title string) + (field width int) + (field height int) + (field format string) + (field duration long) + (field size long) + (field bitrate int) + (field person (array string)) + (field player int) + (field copyright string))) + ) diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java new file mode 100644 index 0000000..c8a88ac --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java @@ -0,0 +1,68 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import org.msgpack.*; +import serializers.ObjectSerializer; + +public class MessagePackDynamicSerializer implements ObjectSerializer +{ + public String getName() { + return "msgpack-dynamic"; + } + + public Object create() throws Exception { + ArrayList media = new ArrayList(11); + media.add("http://javaone.com/keynote.mpg"); + media.add("video/mpg4"); + media.add("Javaone Keynote"); + media.add(1234567L); + media.add(0); + ArrayList person = new ArrayList(2); + person.add("Bill Gates"); + person.add("Steve Jobs"); + media.add(person); + media.add(0); + media.add(0); + media.add(0); + media.add(123L); + media.add(""); + + ArrayList image1 = new ArrayList(5); + image1.add("http://javaone.com/keynote_large.jpg"); + image1.add(0); + image1.add(0); + image1.add(2); + image1.add("Javaone Keynote"); + + ArrayList image2 = new ArrayList(5); + image2.add("http://javaone.com/keynote_thumbnail.jpg"); + image2.add(0); + image2.add(0); + image2.add(1); + image2.add("Javaone Keynote"); + + ArrayList content = new ArrayList(2); + content.add(media); + ArrayList images = new ArrayList(2); + images.add(image1); + images.add(image2); + content.add(images); + + return content; + } + + public byte[] serialize(Object content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.pack(content); + return os.toByteArray(); + } + + public Object deserialize(byte[] array) throws Exception { + UnbufferedUnpacker pac = new UnbufferedUnpacker(); + pac.execute(array); + return (Object)pac.getData(); + } +} + diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java new file mode 100644 index 0000000..4935899 --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java @@ -0,0 +1,70 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import org.msgpack.*; +import serializers.ObjectSerializer; + +public class MessagePackGenericSerializer implements ObjectSerializer +{ + private static final Schema MEDIA_CONTENT_SCHEMA = Schema.parse("(class MediaContent (package serializers.msgpack) (field image (array (class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); + + public String getName() { + return "msgpack-generic"; + } + + public Object create() throws Exception { + HashMap media = new HashMap(11); + media.put("uri", "http://javaone.com/keynote.mpg"); + media.put("format", "video/mpg4"); + media.put("title", "Javaone Keynote"); + media.put("duration", 1234567L); + media.put("bitrate", 0); + ArrayList person = new ArrayList(2); + person.add("Bill Gates"); + person.add("Steve Jobs"); + media.put("person", person); + media.put("player", 0); + media.put("height", 0); + media.put("width", 0); + media.put("size", 123L); + media.put("copyright", ""); + + HashMap image1 = new HashMap(5); + image1.put("uri", "http://javaone.com/keynote_large.jpg"); + image1.put("width", 0); + image1.put("height", 0); + image1.put("size", 2); + image1.put("title", "Javaone Keynote"); + + HashMap image2 = new HashMap(5); + image2.put("uri", "http://javaone.com/keynote_thumbnail.jpg"); + image2.put("width", 0); + image2.put("height", 0); + image2.put("size", 1); + image2.put("title", "Javaone Keynote"); + + HashMap content = new HashMap(2); + content.put("media", media); + ArrayList images = new ArrayList(2); + images.add(image1); + images.add(image2); + content.put("image", images); + + return content; + } + + public byte[] serialize(Object content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.packWithSchema(content, MEDIA_CONTENT_SCHEMA); + return os.toByteArray(); + } + + public Object deserialize(byte[] array) throws Exception { + UnbufferedUnpacker pac = new UnbufferedUnpacker().useSchema(MEDIA_CONTENT_SCHEMA); + pac.execute(array); + return (Object)pac.getData(); + } +} + diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java new file mode 100644 index 0000000..2767474 --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java @@ -0,0 +1,67 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import org.msgpack.*; +import serializers.ObjectSerializer; + +public class MessagePackIndirectSerializer implements ObjectSerializer +{ + public String getName() { + return "msgpack-indirect"; + } + + public MediaContent create() throws Exception { + Media media = new Media(); + media.uri = "http://javaone.com/keynote.mpg"; + media.format = "video/mpg4"; + media.title = "Javaone Keynote"; + media.duration = 1234567L; + media.bitrate = 0; + media.person = new ArrayList(2); + media.person.add("Bill Gates"); + media.person.add("Steve Jobs"); + media.player = 0; + media.height = 0; + media.width = 0; + media.size = 123L; + media.copyright = ""; + + Image image1 = new Image(); + image1.uri = "http://javaone.com/keynote_large.jpg"; + image1.width = 0; + image1.height = 0; + image1.size = 2; + image1.title = "Javaone Keynote"; + + Image image2 = new Image(); + image2.uri = "http://javaone.com/keynote_thumbnail.jpg"; + image2.width = 0; + image2.height = 0; + image2.size = 1; + image2.title = "Javaone Keynote"; + + MediaContent content = new MediaContent(); + content.media = media; + content.image = new ArrayList(2); + content.image.add(image1); + content.image.add(image2); + + return content; + } + + public byte[] serialize(MediaContent content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.pack(content); + return os.toByteArray(); + } + + public MediaContent deserialize(byte[] array) throws Exception { + UnbufferedUnpacker pac = new UnbufferedUnpacker(); + pac.execute(array); + Object obj = pac.getData(); + return (MediaContent)MediaContent.getSchema().convert(obj); + } +} + diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java new file mode 100644 index 0000000..91ded5c --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java @@ -0,0 +1,66 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import org.msgpack.*; +import serializers.ObjectSerializer; + +public class MessagePackSpecificSerializer implements ObjectSerializer +{ + public String getName() { + return "msgpack-specific"; + } + + public MediaContent create() throws Exception { + Media media = new Media(); + media.uri = "http://javaone.com/keynote.mpg"; + media.format = "video/mpg4"; + media.title = "Javaone Keynote"; + media.duration = 1234567L; + media.bitrate = 0; + media.person = new ArrayList(2); + media.person.add("Bill Gates"); + media.person.add("Steve Jobs"); + media.player = 0; + media.height = 0; + media.width = 0; + media.size = 123L; + media.copyright = ""; + + Image image1 = new Image(); + image1.uri = "http://javaone.com/keynote_large.jpg"; + image1.width = 0; + image1.height = 0; + image1.size = 2; + image1.title = "Javaone Keynote"; + + Image image2 = new Image(); + image2.uri = "http://javaone.com/keynote_thumbnail.jpg"; + image2.width = 0; + image2.height = 0; + image2.size = 1; + image2.title = "Javaone Keynote"; + + MediaContent content = new MediaContent(); + content.media = media; + content.image = new ArrayList(2); + content.image.add(image1); + content.image.add(image2); + + return content; + } + + public byte[] serialize(MediaContent content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.pack(content); + return os.toByteArray(); + } + + public MediaContent deserialize(byte[] array) throws Exception { + UnbufferedUnpacker pac = new UnbufferedUnpacker().useSchema(MediaContent.getSchema()); + pac.execute(array); + return (MediaContent)pac.getData(); + } +} + From 0ae1965f6b492efb71f7457ba1dec48ae6110399 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Dec 2009 04:32:33 +0900 Subject: [PATCH 0315/1172] ruby: fixes MessagePack_Unpacker_mark marks uninitialized map_key --- ruby/unpack.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ruby/unpack.c b/ruby/unpack.c index 411a94d..e9d6494 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -127,6 +127,15 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha static VALUE cUnpacker; static VALUE eUnpackError; +// FIXME slow operation +static void init_stack(msgpack_unpack_t* mp) +{ + size_t i; + for(i=0; i < MSGPACK_MAX_STACK_SIZE; ++i) { + mp->stack[i].map_key = Qnil; /* GC */ + } +} + static void MessagePack_Unpacker_free(void* data) { if(data) { free(data); } @@ -137,7 +146,7 @@ static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp) unsigned int i; for(i=0; i < mp->top; ++i) { rb_gc_mark(mp->stack[i].obj); - rb_gc_mark(mp->stack[i].map_key); + rb_gc_mark(mp->stack[i].map_key); /* maybe map_key is not initialized */ } } @@ -154,6 +163,7 @@ static VALUE MessagePack_Unpacker_reset(VALUE self) { UNPACKER(self, mp); template_init(mp); + init_stack(mp); unpack_user u = {0, Qnil}; mp->user = u; return self; @@ -281,6 +291,7 @@ static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) msgpack_unpack_t mp; template_init(&mp); + init_stack(&mp); unpack_user u = {0, Qnil}; mp.user = u; From 7ce866ad7c2884f0b01ee77d99d2f9e01217fcdf Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Dec 2009 06:19:53 +0900 Subject: [PATCH 0316/1172] msgpack template: architecture specific endian conversion --- c/object.c | 10 ++ c/object.h | 4 +- c/pack.h | 4 +- c/unpack.c | 35 ++++-- c/unpack.h | 6 +- c/vrefbuffer.h | 9 ++ cpp/object.hpp | 1 - cpp/pack.hpp | 3 +- msgpack/pack_define.h | 3 +- msgpack/pack_template.h | 253 +++++++++++++++----------------------- msgpack/unpack_define.h | 39 +----- msgpack/unpack_template.h | 6 +- 12 files changed, 154 insertions(+), 219 deletions(-) diff --git a/c/object.c b/c/object.c index bcb2537..a22ce21 100644 --- a/c/object.c +++ b/c/object.c @@ -18,7 +18,17 @@ #include "msgpack/object.h" #include "msgpack/pack.h" #include + +#ifndef _MSC_VER #include +#else +#ifndef PRIu64 +#define PRIu64 "I64u" +#endif +#ifndef PRIi64 +#define PRIi64 "I64d" +#endif +#endif int msgpack_pack_object(msgpack_packer* pk, msgpack_object d) diff --git a/c/object.h b/c/object.h index 7c603b3..0aed0e4 100644 --- a/c/object.h +++ b/c/object.h @@ -19,9 +19,7 @@ #define MSGPACK_OBJECT_H__ #include "msgpack/zone.h" -#include -#include -#include +#include "msgpack/sys.h" #include #ifdef __cplusplus diff --git a/c/pack.h b/c/pack.h index 1a57ea4..1525e0f 100644 --- a/c/pack.h +++ b/c/pack.h @@ -18,11 +18,9 @@ #ifndef MSGPACK_PACK_H__ #define MSGPACK_PACK_H__ -#include -#include -#include #include "msgpack/pack_define.h" #include "msgpack/object.h" +#include #ifdef __cplusplus extern "C" { diff --git a/c/unpack.c b/c/unpack.c index 08fd6cb..4d9af9e 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -101,7 +101,7 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac { o->type = MSGPACK_OBJECT_ARRAY; o->via.array.size = 0; - o->via.array.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); + o->via.array.ptr = (msgpack_object*)msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); if(o->via.array.ptr == NULL) { return -1; } return 0; } @@ -142,30 +142,47 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha #define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced -static const size_t COUNTER_SIZE = sizeof(unsigned int); +#ifndef _MSC_VER +typedef unsigned int counter_t; +#else +typedef long counter_t; +#endif + +#define COUNTER_SIZE (sizeof(volatile counter_t)) + static inline void init_count(void* buffer) { - *(volatile unsigned int*)buffer = 1; + *(volatile counter_t*)buffer = 1; } static inline void decl_count(void* buffer) { - //if(--*(unsigned int*)buffer == 0) { - if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { + // atomic if(--*(counter_t*)buffer == 0) { free(buffer); } + if( +#ifndef _MSC_VER + __sync_sub_and_fetch((counter_t*)buffer, 1) == 0 +#else + InterlockedDecrement((volatile counter_t*)&buffer) == 0 +#endif + ) { free(buffer); } } static inline void incr_count(void* buffer) { - //++*(unsigned int*)buffer; - __sync_add_and_fetch((unsigned int*)buffer, 1); + // atomic ++*(counter_t*)buffer; +#ifndef _MSC_VER + __sync_add_and_fetch((counter_t*)buffer, 1); +#else + InterlockedIncrement((volatile counter_t*)&buffer); +#endif } -static inline unsigned int get_count(void* buffer) +static inline counter_t get_count(void* buffer) { - return *(volatile unsigned int*)buffer; + return *(volatile counter_t*)buffer; } diff --git a/c/unpack.h b/c/unpack.h index ef63774..e17d0d8 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -15,13 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef msgpack_unpacker_H__ -#define msgpack_unpacker_H__ +#ifndef MSGPACK_UNPACKER_H__ +#define MSGPACK_UNPACKER_H__ #include "msgpack/zone.h" #include "msgpack/object.h" -#include -#include #ifdef __cplusplus extern "C" { diff --git a/c/vrefbuffer.h b/c/vrefbuffer.h index baa7c03..063075f 100644 --- a/c/vrefbuffer.h +++ b/c/vrefbuffer.h @@ -19,7 +19,16 @@ #define MSGPACK_VREFBUFFER_H__ #include "msgpack/zone.h" + +#ifndef _WIN32 #include +#else +struct iovec { + void *iov_base; + size_t iov_len; +}; +#endif + #ifdef __cplusplus extern "C" { diff --git a/cpp/object.hpp b/cpp/object.hpp index 9ee575f..ed2e290 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -20,7 +20,6 @@ #include "msgpack/object.h" #include "msgpack/pack.hpp" -#include #include #include #include diff --git a/cpp/pack.hpp b/cpp/pack.hpp index c8e37eb..9c291c1 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -18,10 +18,9 @@ #ifndef MSGPACK_PACK_HPP__ #define MSGPACK_PACK_HPP__ -#include // __BYTE_ORDER +#include "msgpack/pack_define.h" #include #include -#include "msgpack/pack_define.h" namespace msgpack { diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h index 33408e5..aef9295 100644 --- a/msgpack/pack_define.h +++ b/msgpack/pack_define.h @@ -18,8 +18,7 @@ #ifndef MSGPACK_PACK_DEFINE_H__ #define MSGPACK_PACK_DEFINE_H__ -#include -#include +#include "msgpack/sys.h" #include #endif /* msgpack/pack_define.h */ diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index aa620f5..ffbbbba 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -16,88 +16,16 @@ * limitations under the License. */ -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - - #ifdef __LITTLE_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[0] +#define TAKE8_32(d) ((uint8_t*)&d)[0] +#define TAKE8_64(d) ((uint8_t*)&d)[0] #elif __BIG_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[1] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[3] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[7] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[1] +#define TAKE8_32(d) ((uint8_t*)&d)[3] +#define TAKE8_64(d) ((uint8_t*)&d)[7] #endif #ifndef msgpack_pack_inline_func @@ -121,10 +49,10 @@ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } while(0) @@ -133,14 +61,15 @@ do { \ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -150,20 +79,22 @@ do { \ if(d < (1<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -174,24 +105,27 @@ do { \ if(d < (1ULL<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -201,11 +135,11 @@ do { \ do { \ if(d < -(1<<5)) { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } \ } while(0) @@ -214,24 +148,26 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -242,32 +178,36 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<15)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -279,46 +219,52 @@ do { \ if(d < -(1LL<<15)) { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xd3; *(uint64_t*)&buf[1] = msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ if(d < (1LL<<16)) { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -330,49 +276,55 @@ do { \ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) { - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) { - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xd3; *(uint64_t*)&buf[1] = msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } @@ -604,7 +556,8 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { union { char buf[4]; uint32_t num; } f; *((float*)&f.buf) = d; // FIXME - const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; + unsigned char buf[5]; + buf[0] = 0xca; *(uint32_t*)&buf[1] = msgpack_be32(f.num); msgpack_pack_append_buffer(x, buf, 5); } @@ -612,7 +565,8 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { union { char buf[8]; uint64_t num; } f; *((double*)&f.buf) = d; // FIXME - const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; + unsigned char buf[9]; + buf[0] = 0xcb; *(uint64_t*)&buf[1] = msgpack_be64(f.num); msgpack_pack_append_buffer(x, buf, 9); } @@ -655,12 +609,12 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) unsigned char d = 0x90 | n; msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xdc; *(uint16_t*)&buf[1] = msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdd; *(uint32_t*)&buf[1] = msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -674,14 +628,14 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xde, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xde; *(uint16_t*)&buf[1] = msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdf; *(uint32_t*)&buf[1] = msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -695,14 +649,14 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(l < 65536) { - uint16_t d = (uint16_t)l; - unsigned char buf[3] = {0xda, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xda; *(uint16_t*)&buf[1] = msgpack_be16(l); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)l; - unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdb; *(uint32_t*)&buf[1] = msgpack_be32(l); msgpack_pack_append_buffer(x, buf, 5); } } @@ -716,19 +670,10 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer -#undef STORE8_BE8 - -#undef STORE16_BE8 -#undef STORE16_BE16 - -#undef STORE32_BE8 -#undef STORE32_BE16 -#undef STORE32_BE32 - -#undef STORE64_BE8 -#undef STORE64_BE16 -#undef STORE64_BE32 -#undef STORE64_BE64 +#undef TAKE8_8 +#undef TAKE8_16 +#undef TAKE8_32 +#undef TAKE8_64 #undef msgpack_pack_real_uint8 #undef msgpack_pack_real_uint16 diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 63668c2..027d409 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -18,14 +18,10 @@ #ifndef MSGPACK_UNPACK_DEFINE_H__ #define MSGPACK_UNPACK_DEFINE_H__ -#include -#include +#include "msgpack/sys.h" #include #include #include -#ifndef __WIN32__ -#include -#endif #ifdef __cplusplus extern "C" { @@ -37,39 +33,6 @@ extern "C" { #endif -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - -#define msgpack_betoh16(x) ntohs(x) -#define msgpack_betoh32(x) ntohl(x) - -#ifdef __LITTLE_ENDIAN__ -#if defined(__bswap_64) -# define msgpack_betoh64(x) __bswap_64(x) -#elif defined(__DARWIN_OSSwapInt64) -# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) -#else -static inline uint64_t msgpack_betoh64(uint64_t x) { - return ((x << 56) & 0xff00000000000000ULL ) | - ((x << 40) & 0x00ff000000000000ULL ) | - ((x << 24) & 0x0000ff0000000000ULL ) | - ((x << 8) & 0x000000ff00000000ULL ) | - ((x >> 8) & 0x00000000ff000000ULL ) | - ((x >> 24) & 0x0000000000ff0000ULL ) | - ((x >> 40) & 0x000000000000ff00ULL ) | - ((x >> 56) & 0x00000000000000ffULL ) ; -} -#endif -#else -#define msgpack_betoh64(x) (x) -#endif - - typedef enum { CS_HEADER = 0x00, // nil diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index d67fd1e..212a47e 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -126,9 +126,9 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ((unsigned int)*p & 0x1f) #define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) +#define PTR_CAST_16(ptr) msgpack_be16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) msgpack_be32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) msgpack_be64(*(uint64_t*)ptr) if(p == pe) { goto _out; } do { From ba3ba0367cefdb2e87425f257711865e2e540adc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Dec 2009 06:40:29 +0900 Subject: [PATCH 0317/1172] msgpack template: macros for compilers that doesn't not support Case Ranges --- msgpack/unpack_template.h | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 212a47e..6f99d54 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -40,6 +40,11 @@ #error msgpack_unpack_user type is not defined #endif +#ifndef USE_CASE_RANGE +#if !defined(_MSC_VER) +#define USE_CASE_RANGE +#endif +#endif msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; @@ -130,16 +135,28 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c #define PTR_CAST_32(ptr) msgpack_be32(*(uint32_t*)ptr) #define PTR_CAST_64(ptr) msgpack_be64(*(uint64_t*)ptr) +#ifdef USE_CASE_RANGE +#define SWITCH_RANGE_BEGIN switch(*p) { +#define SWITCH_RANGE(FROM, TO) case FROM ... TO: +#define SWITCH_RANGE_DEFAULT default: +#define SWITCH_RANGE_END } +#else +#define SWITCH_RANGE_BEGIN { if(0) { +#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) { +#define SWITCH_RANGE_DEFAULT } else { +#define SWITCH_RANGE_END } } +#endif + if(p == pe) { goto _out; } do { switch(cs) { case CS_HEADER: - switch(*p) { - case 0x00 ... 0x7f: // Positive Fixnum + SWITCH_RANGE_BEGIN + SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum push_fixed_value(_uint8, *(uint8_t*)p); - case 0xe0 ... 0xff: // Negative Fixnum + SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum push_fixed_value(_int8, *(int8_t*)p); - case 0xc0 ... 0xdf: // Variable + SWITCH_RANGE(0xc0, 0xdf) // Variable switch(*p) { case 0xc0: // nil push_simple_value(_nil); @@ -182,16 +199,16 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c default: goto _failed; } - case 0xa0 ... 0xbf: // FixRaw + SWITCH_RANGE(0xa0, 0xbf) // FixRaw again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); - case 0x90 ... 0x9f: // FixArray + SWITCH_RANGE(0x90, 0x9f) // FixArray start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); - case 0x80 ... 0x8f: // FixMap + SWITCH_RANGE(0x80, 0x8f) // FixMap start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); - default: + SWITCH_RANGE_DEFAULT goto _failed; - } + SWITCH_RANGE_END // end CS_HEADER From 35929b46ae1ed8cd001d8b0964ec3a1bb1c053e1 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Dec 2009 07:22:39 +0900 Subject: [PATCH 0318/1172] add msgpack/sysdep.h --- Makefile.am | 3 +- c/object.h | 2 +- c/unpack.c | 33 ++++---------- configure.in | 2 +- msgpack/pack_define.h | 2 +- msgpack/pack_template.h | 64 +++++++++++++------------- msgpack/sysdep.h | 94 +++++++++++++++++++++++++++++++++++++++ msgpack/unpack_define.h | 2 +- msgpack/unpack_template.h | 6 +-- 9 files changed, 143 insertions(+), 65 deletions(-) create mode 100644 msgpack/sysdep.h diff --git a/Makefile.am b/Makefile.am index be3d75f..42fb233 100644 --- a/Makefile.am +++ b/Makefile.am @@ -8,5 +8,6 @@ nobase_include_HEADERS = \ msgpack/pack_define.h \ msgpack/pack_template.h \ msgpack/unpack_define.h \ - msgpack/unpack_template.h + msgpack/unpack_template.h \ + msgpack/sysdep.h diff --git a/c/object.h b/c/object.h index 0aed0e4..27b593e 100644 --- a/c/object.h +++ b/c/object.h @@ -19,7 +19,7 @@ #define MSGPACK_OBJECT_H__ #include "msgpack/zone.h" -#include "msgpack/sys.h" +#include "msgpack/sysdep.h" #include #ifdef __cplusplus diff --git a/c/unpack.c b/c/unpack.c index 4d9af9e..6a435ba 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -141,48 +141,31 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha #define CTX_CAST(m) ((template_context*)(m)) #define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced - -#ifndef _MSC_VER -typedef unsigned int counter_t; -#else -typedef long counter_t; -#endif - -#define COUNTER_SIZE (sizeof(volatile counter_t)) +#define COUNTER_SIZE (sizeof(_msgpack_atomic_counter_t)) static inline void init_count(void* buffer) { - *(volatile counter_t*)buffer = 1; + *(volatile _msgpack_atomic_counter_t*)buffer = 1; } static inline void decl_count(void* buffer) { - // atomic if(--*(counter_t*)buffer == 0) { free(buffer); } - if( -#ifndef _MSC_VER - __sync_sub_and_fetch((counter_t*)buffer, 1) == 0 -#else - InterlockedDecrement((volatile counter_t*)&buffer) == 0 -#endif - ) { + // atomic if(--*(_msgpack_atomic_counter_t*)buffer == 0) { free(buffer); } + if(_msgpack_sync_decr_and_fetch((volatile _msgpack_atomic_counter_t*)buffer)) { free(buffer); } } static inline void incr_count(void* buffer) { - // atomic ++*(counter_t*)buffer; -#ifndef _MSC_VER - __sync_add_and_fetch((counter_t*)buffer, 1); -#else - InterlockedIncrement((volatile counter_t*)&buffer); -#endif + // atomic ++*(_msgpack_atomic_counter_t*)buffer; + _msgpack_sync_incr_and_fetch((volatile _msgpack_atomic_counter_t*)buffer); } -static inline counter_t get_count(void* buffer) +static inline _msgpack_atomic_counter_t get_count(void* buffer) { - return *(volatile counter_t*)buffer; + return *(volatile _msgpack_atomic_counter_t*)buffer; } diff --git a/configure.in b/configure.in index c2ca872..76bd7e4 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.8) +AM_INIT_AUTOMAKE(msgpack, 0.3.9) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h index aef9295..692ef7d 100644 --- a/msgpack/pack_define.h +++ b/msgpack/pack_define.h @@ -18,7 +18,7 @@ #ifndef MSGPACK_PACK_DEFINE_H__ #define MSGPACK_PACK_DEFINE_H__ -#include "msgpack/sys.h" +#include "msgpack/sysdep.h" #include #endif /* msgpack/pack_define.h */ diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index ffbbbba..de148bf 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -69,7 +69,7 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -89,12 +89,12 @@ do { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -115,17 +115,17 @@ do { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -149,7 +149,7 @@ do { \ if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -167,7 +167,7 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -179,12 +179,12 @@ do { \ if(d < -(1<<15)) { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -202,12 +202,12 @@ do { \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -220,19 +220,19 @@ do { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xd3; *(uint64_t*)&buf[1] = msgpack_be64(d); \ + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -252,19 +252,19 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -283,21 +283,21 @@ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { unsigned char buf[3]; - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { unsigned char buf[5]; - buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { unsigned char buf[9]; - buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } @@ -310,21 +310,21 @@ msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { unsigned char buf[3]; - buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { unsigned char buf[5]; - buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { unsigned char buf[9]; - buf[0] = 0xd3; *(uint64_t*)&buf[1] = msgpack_be64(d); + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } @@ -557,7 +557,7 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) union { char buf[4]; uint32_t num; } f; *((float*)&f.buf) = d; // FIXME unsigned char buf[5]; - buf[0] = 0xca; *(uint32_t*)&buf[1] = msgpack_be32(f.num); + buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(f.num); msgpack_pack_append_buffer(x, buf, 5); } @@ -566,7 +566,7 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) union { char buf[8]; uint64_t num; } f; *((double*)&f.buf) = d; // FIXME unsigned char buf[9]; - buf[0] = 0xcb; *(uint64_t*)&buf[1] = msgpack_be64(f.num); + buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(f.num); msgpack_pack_append_buffer(x, buf, 9); } @@ -610,11 +610,11 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { unsigned char buf[3]; - buf[0] = 0xdc; *(uint16_t*)&buf[1] = msgpack_be16(n); + buf[0] = 0xdc; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdd; *(uint32_t*)&buf[1] = msgpack_be32(n); + buf[0] = 0xdd; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -631,11 +631,11 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(n < 65536) { unsigned char buf[3]; - buf[0] = 0xde; *(uint16_t*)&buf[1] = msgpack_be16(n); + buf[0] = 0xde; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdf; *(uint32_t*)&buf[1] = msgpack_be32(n); + buf[0] = 0xdf; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -652,11 +652,11 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(l < 65536) { unsigned char buf[3]; - buf[0] = 0xda; *(uint16_t*)&buf[1] = msgpack_be16(l); + buf[0] = 0xda; *(uint16_t*)&buf[1] = _msgpack_be16(l); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdb; *(uint32_t*)&buf[1] = msgpack_be32(l); + buf[0] = 0xdb; *(uint32_t*)&buf[1] = _msgpack_be32(l); msgpack_pack_append_buffer(x, buf, 5); } } diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h new file mode 100644 index 0000000..106158e --- /dev/null +++ b/msgpack/sysdep.h @@ -0,0 +1,94 @@ +/* + * MessagePack system dependencies + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_SYSDEP_H__ +#define MSGPACK_SYSDEP_H__ + + +#ifdef _MSC_VER +typedef __int8 int8_t; +typedef unsigned __int8 uint8_t; +typedef __int16 int16_t; +typedef unsigned __int16 uint16_t; +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; +#else +#include +#include +#include +#endif + + +#ifdef _WIN32 +typedef long _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr) +#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr) +#else +typedef unsigned int _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1) +#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1) +#endif + + +#ifdef _WIN32 +#include +#else +#include /* __BYTE_ORDER */ +#endif + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#ifdef __LITTLE_ENDIAN__ + +#define _msgpack_be16(x) ntohs(x) +#define _msgpack_be32(x) ntohl(x) + +#if defined(_byteswap_uint64) +# define _msgpack_be64(x) (_byteswap_uint64(x)) +#elif defined(bswap_64) +# define _msgpack_be64(x) bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define _msgpack_be64(x) __DARWIN_OSSwapInt64(x) +#else +#define _msgpack_be64(x) \ + ( ((((uint64_t)x) << 56) & 0xff00000000000000ULL ) | \ + ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \ + ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \ + ((((uint64_t)x) << 8) & 0x000000ff00000000ULL ) | \ + ((((uint64_t)x) >> 8) & 0x00000000ff000000ULL ) | \ + ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \ + ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \ + ((((uint64_t)x) >> 56) & 0x00000000000000ffULL ) ) +#endif + +#else +#define _msgpack_be16(x) (x) +#define _msgpack_be32(x) (x) +#define _msgpack_be64(x) (x) +#endif + + +#endif /* msgpack/sysdep.h */ + diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 027d409..c36aa51 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -18,7 +18,7 @@ #ifndef MSGPACK_UNPACK_DEFINE_H__ #define MSGPACK_UNPACK_DEFINE_H__ -#include "msgpack/sys.h" +#include "msgpack/sysdep.h" #include #include #include diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 6f99d54..3328ea3 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -131,9 +131,9 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ((unsigned int)*p & 0x1f) #define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) msgpack_be16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) msgpack_be32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) msgpack_be64(*(uint64_t*)ptr) +#define PTR_CAST_16(ptr) _msgpack_be16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) _msgpack_be32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) _msgpack_be64(*(uint64_t*)ptr) #ifdef USE_CASE_RANGE #define SWITCH_RANGE_BEGIN switch(*p) { From 0d44348c7dabdbfee4ac1ee17bfb141dfe8706f2 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 11 Dec 2009 04:13:24 +0900 Subject: [PATCH 0319/1172] ruby: version 0.3.2 --- ruby/gem/Rakefile | 6 +++--- ruby/gengem.sh | 1 + ruby/msgpack.gemspec | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ruby/gem/Rakefile b/ruby/gem/Rakefile index 5445906..f2fd684 100644 --- a/ruby/gem/Rakefile +++ b/ruby/gem/Rakefile @@ -15,9 +15,9 @@ AUTHOR = "FURUHASHI Sadayuki" EMAIL = "frsyuki _at_ users.sourceforge.jp" DESCRIPTION = "Binary-based efficient data interchange format." RUBYFORGE_PROJECT = "msgpack" -HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org" +HOMEPATH = "http://msgpack.sourceforge.jp/" BIN_FILES = %w( ) -VERS = "0.3.1" +VERS = "0.3.2" #REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil REV = nil @@ -44,7 +44,7 @@ spec = Gem::Specification.new do |s| s.name = NAME s.version = VERS s.platform = Gem::Platform::RUBY - s.has_rdoc = true + s.has_rdoc = false s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] s.rdoc_options += RDOC_OPTS + ['--exclude', '^(examples|extras)/'] s.summary = DESCRIPTION diff --git a/ruby/gengem.sh b/ruby/gengem.sh index 0afb8f5..359debf 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -15,6 +15,7 @@ cp ../msgpack/pack_define.h gem/msgpack/ cp ../msgpack/pack_template.h gem/msgpack/ cp ../msgpack/unpack_define.h gem/msgpack/ cp ../msgpack/unpack_template.h gem/msgpack/ +cp ../msgpack/sysdep.h gem/msgpack/ cd gem && rake --trace package diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index f0b5c44..59186a4 100755 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::CURRENT s.name = "msgpack" - s.version = "0.3.1" + s.version = "0.3.2" s.summary = "MessagePack" s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" From 5aa47d667783476277409b06d5829322a801df05 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 16 Dec 2009 03:52:14 +0900 Subject: [PATCH 0320/1172] cpp: zone::push_finalizer supports std::auto_ptr --- cpp/zone.hpp.erb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 8fd14a6..f1e4624 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -21,6 +21,7 @@ #include "msgpack/object.hpp" #include "msgpack/zone.h" #include +#include #include <% GENERATION_LIMIT = 15 %> @@ -38,6 +39,9 @@ public: void push_finalizer(void (*func)(void*), void* data); + template + void push_finalizer(std::auto_ptr obj); + void clear(); <%0.upto(GENERATION_LIMIT) {|i|%> @@ -94,6 +98,15 @@ inline void zone::push_finalizer(void (*func)(void*), void* data) } } +template +inline void zone::push_finalizer(std::auto_ptr obj) +{ + if(!msgpack_zone_push_finalizer(this, &zone::object_destructor, obj.get())) { + throw std::bad_alloc(); + } + obj.release(); +} + inline void zone::clear() { msgpack_zone_clear(this); From 686e8ca0f004004f4b8e10438fe91a48a95e6ff9 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 16 Dec 2009 04:08:36 +0900 Subject: [PATCH 0321/1172] c,cpp: fix unpacker --- c/unpack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c/unpack.c b/c/unpack.c index 6a435ba..d5bcb2d 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -152,7 +152,7 @@ static inline void init_count(void* buffer) static inline void decl_count(void* buffer) { // atomic if(--*(_msgpack_atomic_counter_t*)buffer == 0) { free(buffer); } - if(_msgpack_sync_decr_and_fetch((volatile _msgpack_atomic_counter_t*)buffer)) { + if(_msgpack_sync_decr_and_fetch((volatile _msgpack_atomic_counter_t*)buffer) == 0) { free(buffer); } } From dd18402737bdd12fbf53fe0543e29509f9609f3f Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 16 Dec 2009 22:05:31 +0900 Subject: [PATCH 0322/1172] Fix stream unpacker broken. --- python/msgpack/_msgpack.pyx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index cb95146..9ff0c57 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -233,7 +233,9 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=1024*1024): + def __init__(self, file_like=None, int read_size=0): + if read_size == 0: + read_size = 1024*1024 self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] @@ -309,6 +311,7 @@ cdef class Unpacker(object): self.fill_buffer() ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) if ret == 1: + template_init(&self.ctx) return template_data(&self.ctx) elif ret == 0: if self.file_like is not None: @@ -319,3 +322,10 @@ cdef class Unpacker(object): def __iter__(self): return UnpackIterator(self) + + # for debug. + #def _buf(self): + # return PyString_FromStringAndSize(self.buf, self.buf_tail) + + #def _off(self): + # return self.buf_head From 35a4d612c9f45c238f78f37013a4be4c9d74c13b Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 16 Dec 2009 22:05:31 +0900 Subject: [PATCH 0323/1172] Fix stream unpacker broken. --- msgpack/_msgpack.pyx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index cb95146..9ff0c57 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -233,7 +233,9 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=1024*1024): + def __init__(self, file_like=None, int read_size=0): + if read_size == 0: + read_size = 1024*1024 self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] @@ -309,6 +311,7 @@ cdef class Unpacker(object): self.fill_buffer() ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) if ret == 1: + template_init(&self.ctx) return template_data(&self.ctx) elif ret == 0: if self.file_like is not None: @@ -319,3 +322,10 @@ cdef class Unpacker(object): def __iter__(self): return UnpackIterator(self) + + # for debug. + #def _buf(self): + # return PyString_FromStringAndSize(self.buf, self.buf_tail) + + #def _off(self): + # return self.buf_head From 5ff2c6be74cb61363995d43772c5a52566b19000 Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 16 Dec 2009 22:14:13 +0900 Subject: [PATCH 0324/1172] Fix bug come from previous commit --- python/msgpack/_msgpack.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 9ff0c57..dcabc0f 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -311,8 +311,9 @@ cdef class Unpacker(object): self.fill_buffer() ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) if ret == 1: + o = template_data(&self.ctx) template_init(&self.ctx) - return template_data(&self.ctx) + return o elif ret == 0: if self.file_like is not None: return self.unpack() From bb93f093b6a7c4bbba46fe73f08fd8e8c7325443 Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 16 Dec 2009 22:14:13 +0900 Subject: [PATCH 0325/1172] Fix bug come from previous commit --- msgpack/_msgpack.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 9ff0c57..dcabc0f 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -311,8 +311,9 @@ cdef class Unpacker(object): self.fill_buffer() ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) if ret == 1: + o = template_data(&self.ctx) template_init(&self.ctx) - return template_data(&self.ctx) + return o elif ret == 0: if self.file_like is not None: return self.unpack() From 1ed4236bcf1f96e997dc453b2f88a7c4e385688c Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 16 Dec 2009 22:18:17 +0900 Subject: [PATCH 0326/1172] Make new Python release. --- python/setup.py | 2 +- python/setup_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index b26b9e8..ee37098 100755 --- a/python/setup.py +++ b/python/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension #from Cython.Distutils import build_ext import os -version = '0.1.1' +version = '0.1.2' msgpack_mod = Extension('msgpack._msgpack', #sources=['msgpack/_msgpack.pyx'] diff --git a/python/setup_dev.py b/python/setup_dev.py index e28ef25..abed7cd 100755 --- a/python/setup_dev.py +++ b/python/setup_dev.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.1.1dev' +version = '0.1.2dev' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 52a1ace9eff2e8e7ce57a26ef331a05e1a75ad93 Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 16 Dec 2009 22:18:17 +0900 Subject: [PATCH 0327/1172] Make new Python release. --- setup.py | 2 +- setup_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b26b9e8..ee37098 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension #from Cython.Distutils import build_ext import os -version = '0.1.1' +version = '0.1.2' msgpack_mod = Extension('msgpack._msgpack', #sources=['msgpack/_msgpack.pyx'] diff --git a/setup_dev.py b/setup_dev.py index e28ef25..abed7cd 100755 --- a/setup_dev.py +++ b/setup_dev.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.1.1dev' +version = '0.1.2dev' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 3a5f7f53ff26c73396a211157d3d5c713bfc1b8a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 10:43:01 +0900 Subject: [PATCH 0328/1172] Start 0.2.0 --- python/setup.py | 2 +- python/setup_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index ee37098..c48be14 100755 --- a/python/setup.py +++ b/python/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension #from Cython.Distutils import build_ext import os -version = '0.1.2' +version = '0.2.0dev' msgpack_mod = Extension('msgpack._msgpack', #sources=['msgpack/_msgpack.pyx'] diff --git a/python/setup_dev.py b/python/setup_dev.py index abed7cd..4efc769 100755 --- a/python/setup_dev.py +++ b/python/setup_dev.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.1.2dev' +version = '0.2.0dev' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From fecbeb6d075ab996040c5d1ced3f6b3b6ed9faba Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 10:43:01 +0900 Subject: [PATCH 0329/1172] Start 0.2.0 --- setup.py | 2 +- setup_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index ee37098..c48be14 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension #from Cython.Distutils import build_ext import os -version = '0.1.2' +version = '0.2.0dev' msgpack_mod = Extension('msgpack._msgpack', #sources=['msgpack/_msgpack.pyx'] diff --git a/setup_dev.py b/setup_dev.py index abed7cd..4efc769 100755 --- a/setup_dev.py +++ b/setup_dev.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.1.2dev' +version = '0.2.0dev' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 4d33bd456cb31e0aeb3a88128e2cebe89d116ce1 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 10:43:22 +0900 Subject: [PATCH 0330/1172] Update new headers. --- python/msgpack/pack.h | 1 + python/msgpack/pack_define.h | 3 +- python/msgpack/pack_template.h | 253 ++++++++++++------------------- python/msgpack/sysdep.h | 94 ++++++++++++ python/msgpack/unpack_define.h | 52 +------ python/msgpack/unpack_template.h | 41 +++-- 6 files changed, 225 insertions(+), 219 deletions(-) create mode 100644 python/msgpack/sysdep.h diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index 58f021e..2ae95d1 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -18,6 +18,7 @@ #include #include +#include "sysdep.h" #include "pack_define.h" #ifdef __cplusplus diff --git a/python/msgpack/pack_define.h b/python/msgpack/pack_define.h index 33408e5..f72391b 100644 --- a/python/msgpack/pack_define.h +++ b/python/msgpack/pack_define.h @@ -18,8 +18,7 @@ #ifndef MSGPACK_PACK_DEFINE_H__ #define MSGPACK_PACK_DEFINE_H__ -#include -#include +#include "sysdep.h" #include #endif /* msgpack/pack_define.h */ diff --git a/python/msgpack/pack_template.h b/python/msgpack/pack_template.h index aa620f5..de148bf 100644 --- a/python/msgpack/pack_template.h +++ b/python/msgpack/pack_template.h @@ -16,88 +16,16 @@ * limitations under the License. */ -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - - #ifdef __LITTLE_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[0] +#define TAKE8_32(d) ((uint8_t*)&d)[0] +#define TAKE8_64(d) ((uint8_t*)&d)[0] #elif __BIG_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[1] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[3] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[7] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[1] +#define TAKE8_32(d) ((uint8_t*)&d)[3] +#define TAKE8_64(d) ((uint8_t*)&d)[7] #endif #ifndef msgpack_pack_inline_func @@ -121,10 +49,10 @@ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } while(0) @@ -133,14 +61,15 @@ do { \ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -150,20 +79,22 @@ do { \ if(d < (1<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -174,24 +105,27 @@ do { \ if(d < (1ULL<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -201,11 +135,11 @@ do { \ do { \ if(d < -(1<<5)) { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } \ } while(0) @@ -214,24 +148,26 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -242,32 +178,36 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<15)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -279,46 +219,52 @@ do { \ if(d < -(1LL<<15)) { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ if(d < (1LL<<16)) { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -330,49 +276,55 @@ do { \ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) { - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) { - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } @@ -604,7 +556,8 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { union { char buf[4]; uint32_t num; } f; *((float*)&f.buf) = d; // FIXME - const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; + unsigned char buf[5]; + buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(f.num); msgpack_pack_append_buffer(x, buf, 5); } @@ -612,7 +565,8 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { union { char buf[8]; uint64_t num; } f; *((double*)&f.buf) = d; // FIXME - const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; + unsigned char buf[9]; + buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(f.num); msgpack_pack_append_buffer(x, buf, 9); } @@ -655,12 +609,12 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) unsigned char d = 0x90 | n; msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xdc; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdd; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -674,14 +628,14 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xde, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xde; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdf; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -695,14 +649,14 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(l < 65536) { - uint16_t d = (uint16_t)l; - unsigned char buf[3] = {0xda, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xda; *(uint16_t*)&buf[1] = _msgpack_be16(l); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)l; - unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdb; *(uint32_t*)&buf[1] = _msgpack_be32(l); msgpack_pack_append_buffer(x, buf, 5); } } @@ -716,19 +670,10 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer -#undef STORE8_BE8 - -#undef STORE16_BE8 -#undef STORE16_BE16 - -#undef STORE32_BE8 -#undef STORE32_BE16 -#undef STORE32_BE32 - -#undef STORE64_BE8 -#undef STORE64_BE16 -#undef STORE64_BE32 -#undef STORE64_BE64 +#undef TAKE8_8 +#undef TAKE8_16 +#undef TAKE8_32 +#undef TAKE8_64 #undef msgpack_pack_real_uint8 #undef msgpack_pack_real_uint16 diff --git a/python/msgpack/sysdep.h b/python/msgpack/sysdep.h new file mode 100644 index 0000000..106158e --- /dev/null +++ b/python/msgpack/sysdep.h @@ -0,0 +1,94 @@ +/* + * MessagePack system dependencies + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_SYSDEP_H__ +#define MSGPACK_SYSDEP_H__ + + +#ifdef _MSC_VER +typedef __int8 int8_t; +typedef unsigned __int8 uint8_t; +typedef __int16 int16_t; +typedef unsigned __int16 uint16_t; +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; +#else +#include +#include +#include +#endif + + +#ifdef _WIN32 +typedef long _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr) +#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr) +#else +typedef unsigned int _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1) +#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1) +#endif + + +#ifdef _WIN32 +#include +#else +#include /* __BYTE_ORDER */ +#endif + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#ifdef __LITTLE_ENDIAN__ + +#define _msgpack_be16(x) ntohs(x) +#define _msgpack_be32(x) ntohl(x) + +#if defined(_byteswap_uint64) +# define _msgpack_be64(x) (_byteswap_uint64(x)) +#elif defined(bswap_64) +# define _msgpack_be64(x) bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define _msgpack_be64(x) __DARWIN_OSSwapInt64(x) +#else +#define _msgpack_be64(x) \ + ( ((((uint64_t)x) << 56) & 0xff00000000000000ULL ) | \ + ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \ + ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \ + ((((uint64_t)x) << 8) & 0x000000ff00000000ULL ) | \ + ((((uint64_t)x) >> 8) & 0x00000000ff000000ULL ) | \ + ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \ + ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \ + ((((uint64_t)x) >> 56) & 0x00000000000000ffULL ) ) +#endif + +#else +#define _msgpack_be16(x) (x) +#define _msgpack_be32(x) (x) +#define _msgpack_be64(x) (x) +#endif + + +#endif /* msgpack/sysdep.h */ + diff --git a/python/msgpack/unpack_define.h b/python/msgpack/unpack_define.h index d997569..63d90a8 100644 --- a/python/msgpack/unpack_define.h +++ b/python/msgpack/unpack_define.h @@ -18,14 +18,10 @@ #ifndef MSGPACK_UNPACK_DEFINE_H__ #define MSGPACK_UNPACK_DEFINE_H__ -#include -#include +#include "sysdep.h" #include #include #include -#ifndef __WIN32__ -#include -#endif #ifdef __cplusplus extern "C" { @@ -37,52 +33,6 @@ extern "C" { #endif -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - -#ifdef __WIN32__ -static inline uint16_t msgpack_betoh16(uint16_t x) { - return ((x << 8) & 0xff00U) | - ((x >> 8) & 0x00ffU); -} -static inline uint32_t msgpack_betoh32(uint32_t x) { - return ((x << 24) & 0xff000000UL ) | - ((x << 8) & 0x00ff0000UL ) | - ((x >> 8) & 0x0000ff00UL ) | - ((x >> 24) & 0x000000ffUL ); -} -#else -#define msgpack_betoh16(x) ntohs(x) -#define msgpack_betoh32(x) ntohl(x) -#endif - -#ifdef __LITTLE_ENDIAN__ -#if defined(__bswap_64) -# define msgpack_betoh64(x) __bswap_64(x) -#elif defined(__DARWIN_OSSwapInt64) -# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) -#else -static inline uint64_t msgpack_betoh64(uint64_t x) { - return ((x << 56) & 0xff00000000000000ULL ) | - ((x << 40) & 0x00ff000000000000ULL ) | - ((x << 24) & 0x0000ff0000000000ULL ) | - ((x << 8) & 0x000000ff00000000ULL ) | - ((x >> 8) & 0x00000000ff000000ULL ) | - ((x >> 24) & 0x0000000000ff0000ULL ) | - ((x >> 40) & 0x000000000000ff00ULL ) | - ((x >> 56) & 0x00000000000000ffULL ) ; -} -#endif -#else -#define msgpack_betoh64(x) (x) -#endif - - typedef enum { CS_HEADER = 0x00, // nil diff --git a/python/msgpack/unpack_template.h b/python/msgpack/unpack_template.h index c960c3a..ca6e1f3 100644 --- a/python/msgpack/unpack_template.h +++ b/python/msgpack/unpack_template.h @@ -40,6 +40,11 @@ #error msgpack_unpack_user type is not defined #endif +#ifndef USE_CASE_RANGE +#if !defined(_MSC_VER) +#define USE_CASE_RANGE +#endif +#endif msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; @@ -131,20 +136,32 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ((unsigned int)*p & 0x1f) #define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) +#define PTR_CAST_16(ptr) _msgpack_be16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) _msgpack_be32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) _msgpack_be64(*(uint64_t*)ptr) + +#ifdef USE_CASE_RANGE +#define SWITCH_RANGE_BEGIN switch(*p) { +#define SWITCH_RANGE(FROM, TO) case FROM ... TO: +#define SWITCH_RANGE_DEFAULT default: +#define SWITCH_RANGE_END } +#else +#define SWITCH_RANGE_BEGIN { if(0) { +#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) { +#define SWITCH_RANGE_DEFAULT } else { +#define SWITCH_RANGE_END } } +#endif if(p == pe) { goto _out; } do { switch(cs) { case CS_HEADER: - switch(*p) { - case 0x00 ... 0x7f: // Positive Fixnum + SWITCH_RANGE_BEGIN + SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum push_fixed_value(_uint8, *(uint8_t*)p); - case 0xe0 ... 0xff: // Negative Fixnum + SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum push_fixed_value(_int8, *(int8_t*)p); - case 0xc0 ... 0xdf: // Variable + SWITCH_RANGE(0xc0, 0xdf) // Variable switch(*p) { case 0xc0: // nil push_simple_value(_nil); @@ -187,16 +204,16 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c default: goto _failed; } - case 0xa0 ... 0xbf: // FixRaw + SWITCH_RANGE(0xa0, 0xbf) // FixRaw again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); - case 0x90 ... 0x9f: // FixArray + SWITCH_RANGE(0x90, 0x9f) // FixArray start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); - case 0x80 ... 0x8f: // FixMap + SWITCH_RANGE(0x80, 0x8f) // FixMap start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); - default: + SWITCH_RANGE_DEFAULT goto _failed; - } + SWITCH_RANGE_END // end CS_HEADER From af7c4d2a60086503ee9c124f8cc79a05c2fabe70 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 10:43:22 +0900 Subject: [PATCH 0331/1172] Update new headers. --- msgpack/pack.h | 1 + msgpack/pack_define.h | 3 +- msgpack/pack_template.h | 253 +++++++++++++++----------------------- msgpack/sysdep.h | 94 ++++++++++++++ msgpack/unpack_define.h | 52 +------- msgpack/unpack_template.h | 41 ++++-- 6 files changed, 225 insertions(+), 219 deletions(-) create mode 100644 msgpack/sysdep.h diff --git a/msgpack/pack.h b/msgpack/pack.h index 58f021e..2ae95d1 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -18,6 +18,7 @@ #include #include +#include "sysdep.h" #include "pack_define.h" #ifdef __cplusplus diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h index 33408e5..f72391b 100644 --- a/msgpack/pack_define.h +++ b/msgpack/pack_define.h @@ -18,8 +18,7 @@ #ifndef MSGPACK_PACK_DEFINE_H__ #define MSGPACK_PACK_DEFINE_H__ -#include -#include +#include "sysdep.h" #include #endif /* msgpack/pack_define.h */ diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index aa620f5..de148bf 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -16,88 +16,16 @@ * limitations under the License. */ -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - - #ifdef __LITTLE_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[0] +#define TAKE8_32(d) ((uint8_t*)&d)[0] +#define TAKE8_64(d) ((uint8_t*)&d)[0] #elif __BIG_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[1] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[3] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[7] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[1] +#define TAKE8_32(d) ((uint8_t*)&d)[3] +#define TAKE8_64(d) ((uint8_t*)&d)[7] #endif #ifndef msgpack_pack_inline_func @@ -121,10 +49,10 @@ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } while(0) @@ -133,14 +61,15 @@ do { \ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -150,20 +79,22 @@ do { \ if(d < (1<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -174,24 +105,27 @@ do { \ if(d < (1ULL<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -201,11 +135,11 @@ do { \ do { \ if(d < -(1<<5)) { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } \ } while(0) @@ -214,24 +148,26 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -242,32 +178,36 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<15)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -279,46 +219,52 @@ do { \ if(d < -(1LL<<15)) { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ if(d < (1LL<<16)) { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -330,49 +276,55 @@ do { \ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) { - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) { - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } @@ -604,7 +556,8 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { union { char buf[4]; uint32_t num; } f; *((float*)&f.buf) = d; // FIXME - const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; + unsigned char buf[5]; + buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(f.num); msgpack_pack_append_buffer(x, buf, 5); } @@ -612,7 +565,8 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { union { char buf[8]; uint64_t num; } f; *((double*)&f.buf) = d; // FIXME - const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; + unsigned char buf[9]; + buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(f.num); msgpack_pack_append_buffer(x, buf, 9); } @@ -655,12 +609,12 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) unsigned char d = 0x90 | n; msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xdc; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdd; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -674,14 +628,14 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xde, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xde; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdf; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -695,14 +649,14 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(l < 65536) { - uint16_t d = (uint16_t)l; - unsigned char buf[3] = {0xda, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xda; *(uint16_t*)&buf[1] = _msgpack_be16(l); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)l; - unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdb; *(uint32_t*)&buf[1] = _msgpack_be32(l); msgpack_pack_append_buffer(x, buf, 5); } } @@ -716,19 +670,10 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer -#undef STORE8_BE8 - -#undef STORE16_BE8 -#undef STORE16_BE16 - -#undef STORE32_BE8 -#undef STORE32_BE16 -#undef STORE32_BE32 - -#undef STORE64_BE8 -#undef STORE64_BE16 -#undef STORE64_BE32 -#undef STORE64_BE64 +#undef TAKE8_8 +#undef TAKE8_16 +#undef TAKE8_32 +#undef TAKE8_64 #undef msgpack_pack_real_uint8 #undef msgpack_pack_real_uint16 diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h new file mode 100644 index 0000000..106158e --- /dev/null +++ b/msgpack/sysdep.h @@ -0,0 +1,94 @@ +/* + * MessagePack system dependencies + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_SYSDEP_H__ +#define MSGPACK_SYSDEP_H__ + + +#ifdef _MSC_VER +typedef __int8 int8_t; +typedef unsigned __int8 uint8_t; +typedef __int16 int16_t; +typedef unsigned __int16 uint16_t; +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; +#else +#include +#include +#include +#endif + + +#ifdef _WIN32 +typedef long _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr) +#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr) +#else +typedef unsigned int _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1) +#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1) +#endif + + +#ifdef _WIN32 +#include +#else +#include /* __BYTE_ORDER */ +#endif + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#ifdef __LITTLE_ENDIAN__ + +#define _msgpack_be16(x) ntohs(x) +#define _msgpack_be32(x) ntohl(x) + +#if defined(_byteswap_uint64) +# define _msgpack_be64(x) (_byteswap_uint64(x)) +#elif defined(bswap_64) +# define _msgpack_be64(x) bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define _msgpack_be64(x) __DARWIN_OSSwapInt64(x) +#else +#define _msgpack_be64(x) \ + ( ((((uint64_t)x) << 56) & 0xff00000000000000ULL ) | \ + ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \ + ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \ + ((((uint64_t)x) << 8) & 0x000000ff00000000ULL ) | \ + ((((uint64_t)x) >> 8) & 0x00000000ff000000ULL ) | \ + ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \ + ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \ + ((((uint64_t)x) >> 56) & 0x00000000000000ffULL ) ) +#endif + +#else +#define _msgpack_be16(x) (x) +#define _msgpack_be32(x) (x) +#define _msgpack_be64(x) (x) +#endif + + +#endif /* msgpack/sysdep.h */ + diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index d997569..63d90a8 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -18,14 +18,10 @@ #ifndef MSGPACK_UNPACK_DEFINE_H__ #define MSGPACK_UNPACK_DEFINE_H__ -#include -#include +#include "sysdep.h" #include #include #include -#ifndef __WIN32__ -#include -#endif #ifdef __cplusplus extern "C" { @@ -37,52 +33,6 @@ extern "C" { #endif -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - -#ifdef __WIN32__ -static inline uint16_t msgpack_betoh16(uint16_t x) { - return ((x << 8) & 0xff00U) | - ((x >> 8) & 0x00ffU); -} -static inline uint32_t msgpack_betoh32(uint32_t x) { - return ((x << 24) & 0xff000000UL ) | - ((x << 8) & 0x00ff0000UL ) | - ((x >> 8) & 0x0000ff00UL ) | - ((x >> 24) & 0x000000ffUL ); -} -#else -#define msgpack_betoh16(x) ntohs(x) -#define msgpack_betoh32(x) ntohl(x) -#endif - -#ifdef __LITTLE_ENDIAN__ -#if defined(__bswap_64) -# define msgpack_betoh64(x) __bswap_64(x) -#elif defined(__DARWIN_OSSwapInt64) -# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) -#else -static inline uint64_t msgpack_betoh64(uint64_t x) { - return ((x << 56) & 0xff00000000000000ULL ) | - ((x << 40) & 0x00ff000000000000ULL ) | - ((x << 24) & 0x0000ff0000000000ULL ) | - ((x << 8) & 0x000000ff00000000ULL ) | - ((x >> 8) & 0x00000000ff000000ULL ) | - ((x >> 24) & 0x0000000000ff0000ULL ) | - ((x >> 40) & 0x000000000000ff00ULL ) | - ((x >> 56) & 0x00000000000000ffULL ) ; -} -#endif -#else -#define msgpack_betoh64(x) (x) -#endif - - typedef enum { CS_HEADER = 0x00, // nil diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index c960c3a..ca6e1f3 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -40,6 +40,11 @@ #error msgpack_unpack_user type is not defined #endif +#ifndef USE_CASE_RANGE +#if !defined(_MSC_VER) +#define USE_CASE_RANGE +#endif +#endif msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; @@ -131,20 +136,32 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ((unsigned int)*p & 0x1f) #define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) +#define PTR_CAST_16(ptr) _msgpack_be16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) _msgpack_be32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) _msgpack_be64(*(uint64_t*)ptr) + +#ifdef USE_CASE_RANGE +#define SWITCH_RANGE_BEGIN switch(*p) { +#define SWITCH_RANGE(FROM, TO) case FROM ... TO: +#define SWITCH_RANGE_DEFAULT default: +#define SWITCH_RANGE_END } +#else +#define SWITCH_RANGE_BEGIN { if(0) { +#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) { +#define SWITCH_RANGE_DEFAULT } else { +#define SWITCH_RANGE_END } } +#endif if(p == pe) { goto _out; } do { switch(cs) { case CS_HEADER: - switch(*p) { - case 0x00 ... 0x7f: // Positive Fixnum + SWITCH_RANGE_BEGIN + SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum push_fixed_value(_uint8, *(uint8_t*)p); - case 0xe0 ... 0xff: // Negative Fixnum + SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum push_fixed_value(_int8, *(int8_t*)p); - case 0xc0 ... 0xdf: // Variable + SWITCH_RANGE(0xc0, 0xdf) // Variable switch(*p) { case 0xc0: // nil push_simple_value(_nil); @@ -187,16 +204,16 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c default: goto _failed; } - case 0xa0 ... 0xbf: // FixRaw + SWITCH_RANGE(0xa0, 0xbf) // FixRaw again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); - case 0x90 ... 0x9f: // FixArray + SWITCH_RANGE(0x90, 0x9f) // FixArray start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); - case 0x80 ... 0x8f: // FixMap + SWITCH_RANGE(0x80, 0x8f) // FixMap start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); - default: + SWITCH_RANGE_DEFAULT goto _failed; - } + SWITCH_RANGE_END // end CS_HEADER From f4c5b15cc6cf972695bb2281714f5b13b299fbd3 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 11:13:47 +0900 Subject: [PATCH 0332/1172] Add `use_tuple` option that returns tuple for array object to Unpacker. --- python/msgpack/_msgpack.pyx | 10 +++++++++- python/msgpack/unpack.h | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index dcabc0f..b20ad7b 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -151,7 +151,11 @@ def packb(object o): packs = packb cdef extern from "unpack.h": + ctypedef struct msgpack_user: + int use_tuple + ctypedef struct template_context: + msgpack_user user PyObject* obj size_t count unsigned int ct @@ -170,6 +174,7 @@ def unpackb(object packed_bytes): cdef size_t off = 0 cdef int ret template_init(&ctx) + ctx.user.use_tuple = 0 ret = template_execute(&ctx, p, len(packed_bytes), &off) if ret == 1: return template_data(&ctx) @@ -225,6 +230,7 @@ cdef class Unpacker(object): cdef object file_like cdef int read_size cdef object waiting_bytes + cdef int use_tuple def __cinit__(self): self.buf = NULL @@ -233,9 +239,10 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=0): + def __init__(self, file_like=None, int read_size=0, use_tuple=0): if read_size == 0: read_size = 1024*1024 + self.use_tuple = use_tuple self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] @@ -244,6 +251,7 @@ cdef class Unpacker(object): self.buf_head = 0 self.buf_tail = 0 template_init(&self.ctx) + self.ctx.user.use_tuple = use_tuple def feed(self, next_bytes): if not isinstance(next_bytes, str): diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 40058d0..0bf2568 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -20,6 +20,7 @@ #include "unpack_define.h" typedef struct unpack_user { + int use_tuple; } unpack_user; @@ -135,7 +136,8 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - PyObject *p = PyList_New(n); + PyObject *p = u->use_tuple ? PyTuple_New(n) : PyList_New(n); + if (!p) return -1; *o = p; @@ -143,7 +145,15 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac } static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) -{ PyList_SET_ITEM(*c, current, o); return 0; } +{ + if (u->use_tuple) { + PyTuple_SET_ITEM(*c, current, o); + } + else { + PyList_SET_ITEM(*c, current, o); + } + return 0; +} static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { From 77e3e59620d65589451c833fa01e9c9e5f39624d Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 11:13:47 +0900 Subject: [PATCH 0333/1172] Add `use_tuple` option that returns tuple for array object to Unpacker. --- msgpack/_msgpack.pyx | 10 +++++++++- msgpack/unpack.h | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index dcabc0f..b20ad7b 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -151,7 +151,11 @@ def packb(object o): packs = packb cdef extern from "unpack.h": + ctypedef struct msgpack_user: + int use_tuple + ctypedef struct template_context: + msgpack_user user PyObject* obj size_t count unsigned int ct @@ -170,6 +174,7 @@ def unpackb(object packed_bytes): cdef size_t off = 0 cdef int ret template_init(&ctx) + ctx.user.use_tuple = 0 ret = template_execute(&ctx, p, len(packed_bytes), &off) if ret == 1: return template_data(&ctx) @@ -225,6 +230,7 @@ cdef class Unpacker(object): cdef object file_like cdef int read_size cdef object waiting_bytes + cdef int use_tuple def __cinit__(self): self.buf = NULL @@ -233,9 +239,10 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=0): + def __init__(self, file_like=None, int read_size=0, use_tuple=0): if read_size == 0: read_size = 1024*1024 + self.use_tuple = use_tuple self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] @@ -244,6 +251,7 @@ cdef class Unpacker(object): self.buf_head = 0 self.buf_tail = 0 template_init(&self.ctx) + self.ctx.user.use_tuple = use_tuple def feed(self, next_bytes): if not isinstance(next_bytes, str): diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 40058d0..0bf2568 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -20,6 +20,7 @@ #include "unpack_define.h" typedef struct unpack_user { + int use_tuple; } unpack_user; @@ -135,7 +136,8 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - PyObject *p = PyList_New(n); + PyObject *p = u->use_tuple ? PyTuple_New(n) : PyList_New(n); + if (!p) return -1; *o = p; @@ -143,7 +145,15 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac } static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) -{ PyList_SET_ITEM(*c, current, o); return 0; } +{ + if (u->use_tuple) { + PyTuple_SET_ITEM(*c, current, o); + } + else { + PyList_SET_ITEM(*c, current, o); + } + return 0; +} static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { From b9f78821d423c2883ddff4d8471cc4c93d032388 Mon Sep 17 00:00:00 2001 From: inada-n Date: Thu, 17 Dec 2009 15:19:18 +0900 Subject: [PATCH 0334/1172] Make tuple default. --- python/msgpack/_msgpack.pyx | 12 ++++++------ python/msgpack/unpack.h | 12 +++++------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index b20ad7b..eb83c85 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -152,7 +152,7 @@ packs = packb cdef extern from "unpack.h": ctypedef struct msgpack_user: - int use_tuple + int use_list ctypedef struct template_context: msgpack_user user @@ -174,7 +174,7 @@ def unpackb(object packed_bytes): cdef size_t off = 0 cdef int ret template_init(&ctx) - ctx.user.use_tuple = 0 + ctx.user.use_list = 0 ret = template_execute(&ctx, p, len(packed_bytes), &off) if ret == 1: return template_data(&ctx) @@ -230,7 +230,7 @@ cdef class Unpacker(object): cdef object file_like cdef int read_size cdef object waiting_bytes - cdef int use_tuple + cdef int use_list def __cinit__(self): self.buf = NULL @@ -239,10 +239,10 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=0, use_tuple=0): + def __init__(self, file_like=None, int read_size=0, use_list=0): if read_size == 0: read_size = 1024*1024 - self.use_tuple = use_tuple + self.use_list = use_list self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] @@ -251,7 +251,7 @@ cdef class Unpacker(object): self.buf_head = 0 self.buf_tail = 0 template_init(&self.ctx) - self.ctx.user.use_tuple = use_tuple + self.ctx.user.use_list = use_list def feed(self, next_bytes): if not isinstance(next_bytes, str): diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 0bf2568..61a3786 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -20,7 +20,7 @@ #include "unpack_define.h" typedef struct unpack_user { - int use_tuple; + int use_list; } unpack_user; @@ -136,7 +136,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - PyObject *p = u->use_tuple ? PyTuple_New(n) : PyList_New(n); + PyObject *p = u->use_list ? PyList_New(n) : PyTuple_New(n); if (!p) return -1; @@ -146,12 +146,10 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) { - if (u->use_tuple) { - PyTuple_SET_ITEM(*c, current, o); - } - else { + if (u->use_list) PyList_SET_ITEM(*c, current, o); - } + else + PyTuple_SET_ITEM(*c, current, o); return 0; } From 658c90f132d9582ec1e8ec68a30f9e67b8d7a572 Mon Sep 17 00:00:00 2001 From: inada-n Date: Thu, 17 Dec 2009 15:19:18 +0900 Subject: [PATCH 0335/1172] Make tuple default. --- msgpack/_msgpack.pyx | 12 ++++++------ msgpack/unpack.h | 12 +++++------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index b20ad7b..eb83c85 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -152,7 +152,7 @@ packs = packb cdef extern from "unpack.h": ctypedef struct msgpack_user: - int use_tuple + int use_list ctypedef struct template_context: msgpack_user user @@ -174,7 +174,7 @@ def unpackb(object packed_bytes): cdef size_t off = 0 cdef int ret template_init(&ctx) - ctx.user.use_tuple = 0 + ctx.user.use_list = 0 ret = template_execute(&ctx, p, len(packed_bytes), &off) if ret == 1: return template_data(&ctx) @@ -230,7 +230,7 @@ cdef class Unpacker(object): cdef object file_like cdef int read_size cdef object waiting_bytes - cdef int use_tuple + cdef int use_list def __cinit__(self): self.buf = NULL @@ -239,10 +239,10 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=0, use_tuple=0): + def __init__(self, file_like=None, int read_size=0, use_list=0): if read_size == 0: read_size = 1024*1024 - self.use_tuple = use_tuple + self.use_list = use_list self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] @@ -251,7 +251,7 @@ cdef class Unpacker(object): self.buf_head = 0 self.buf_tail = 0 template_init(&self.ctx) - self.ctx.user.use_tuple = use_tuple + self.ctx.user.use_list = use_list def feed(self, next_bytes): if not isinstance(next_bytes, str): diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 0bf2568..61a3786 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -20,7 +20,7 @@ #include "unpack_define.h" typedef struct unpack_user { - int use_tuple; + int use_list; } unpack_user; @@ -136,7 +136,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - PyObject *p = u->use_tuple ? PyTuple_New(n) : PyList_New(n); + PyObject *p = u->use_list ? PyList_New(n) : PyTuple_New(n); if (!p) return -1; @@ -146,12 +146,10 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) { - if (u->use_tuple) { - PyTuple_SET_ITEM(*c, current, o); - } - else { + if (u->use_list) PyList_SET_ITEM(*c, current, o); - } + else + PyTuple_SET_ITEM(*c, current, o); return 0; } From 5cf85a82d3a39dd9135de473d0fa4ed21aea8270 Mon Sep 17 00:00:00 2001 From: inada-n Date: Thu, 17 Dec 2009 15:20:20 +0900 Subject: [PATCH 0336/1172] Add .gitignore for Python. --- python/.gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 python/.gitignore diff --git a/python/.gitignore b/python/.gitignore new file mode 100644 index 0000000..81df440 --- /dev/null +++ b/python/.gitignore @@ -0,0 +1,3 @@ +MANIFEST +build/* +dist/* From 8194212d5845d648cc35b677c71db61ceb2d11d3 Mon Sep 17 00:00:00 2001 From: inada-n Date: Thu, 17 Dec 2009 15:20:20 +0900 Subject: [PATCH 0337/1172] Add .gitignore for Python. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..81df440 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +MANIFEST +build/* +dist/* From 63b9a876b0b402709b7b8773b08a58f8fd0bc8b2 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 17:58:41 +0900 Subject: [PATCH 0338/1172] Fix tests. --- python/test/test_case.py | 9 +++++---- python/test/test_format.py | 21 +++++++++++---------- python/test/test_pack.py | 2 +- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/python/test/test_case.py b/python/test/test_case.py index d754fb0..a08c6ce 100644 --- a/python/test/test_case.py +++ b/python/test/test_case.py @@ -5,6 +5,7 @@ from nose import main from nose.tools import * from msgpack import packs, unpacks + def check(length, obj): v = packs(obj) assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) @@ -54,7 +55,7 @@ def test_raw32(): def check_array(overhead, num): - check(num + overhead, [None] * num) + check(num + overhead, (None,) * num) def test_fixarray(): check_array(1, 0) @@ -86,9 +87,9 @@ def test_match(): (-129, '\xd1\xff\x7f'), ({1:1}, '\x81\x01\x01'), (1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"), - ([], '\x90'), - (range(15),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), - (range(16),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), + ((), '\x90'), + (tuple(range(15)),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), + (tuple(range(16)),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), ({}, '\x80'), (dict([(x,x) for x in range(15)]), '\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'), (dict([(x,x) for x in range(16)]), '\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'), diff --git a/python/test/test_format.py b/python/test/test_format.py index 009a764..562ef54 100644 --- a/python/test/test_format.py +++ b/python/test/test_format.py @@ -10,21 +10,21 @@ def check(src, should): def testSimpleValue(): check("\x93\xc0\xc2\xc3", - [None, False, True]) + (None, False, True,)) def testFixnum(): check("\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", - [[0,64,127], [-32,-16,-1]] + ((0,64,127,), (-32,-16,-1,),) ) def testFixArray(): check("\x92\x90\x91\x91\xc0", - [[],[[None]]], + ((),((None,),),), ) def testFixRaw(): check("\x94\xa0\xa1a\xa2bc\xa3def", - ["", "a", "bc", "def"], + ("", "a", "bc", "def",), ) def testFixMap(): @@ -38,25 +38,26 @@ def testUnsignedInt(): "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" "\xce\xff\xff\xff\xff", - [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,), ) def testSignedInt(): check("\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" "\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" "\xd2\xff\xff\xff\xff", - [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1]) + (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,)) def testRaw(): check("\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" "\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", - ["", "a", "ab", "", "a", "ab"]) + ("", "a", "ab", "", "a", "ab")) def testArray(): check("\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" "\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02" "\xc2\xc3", - [[], [None], [False,True], [], [None], [False,True]]) + ((), (None,), (False,True), (), (None,), (False,True)) + ) def testMap(): check( @@ -67,8 +68,8 @@ def testMap(): "\xdf\x00\x00\x00\x00" "\xdf\x00\x00\x00\x01\xc0\xc2" "\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", - [{}, {None: False}, {True: False, None: False}, {}, - {None: False}, {True: False, None: False}]) + ({}, {None: False}, {True: False, None: False}, {}, + {None: False}, {True: False, None: False})) if __name__ == '__main__': main() diff --git a/python/test/test_pack.py b/python/test/test_pack.py index 86badb5..5dec068 100644 --- a/python/test/test_pack.py +++ b/python/test/test_pack.py @@ -17,7 +17,7 @@ def testPack(): 1.0, "", "a", "a"*31, "a"*32, None, True, False, - [], [[]], [[], None], + (), ((),), ((), None,), {None: 0}, (1<<23), ] From 472f5be977e913632cc1d15e663a34b62dfbab5c Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 17:58:41 +0900 Subject: [PATCH 0339/1172] Fix tests. --- test/test_case.py | 9 +++++---- test/test_format.py | 21 +++++++++++---------- test/test_pack.py | 2 +- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/test/test_case.py b/test/test_case.py index d754fb0..a08c6ce 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -5,6 +5,7 @@ from nose import main from nose.tools import * from msgpack import packs, unpacks + def check(length, obj): v = packs(obj) assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) @@ -54,7 +55,7 @@ def test_raw32(): def check_array(overhead, num): - check(num + overhead, [None] * num) + check(num + overhead, (None,) * num) def test_fixarray(): check_array(1, 0) @@ -86,9 +87,9 @@ def test_match(): (-129, '\xd1\xff\x7f'), ({1:1}, '\x81\x01\x01'), (1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"), - ([], '\x90'), - (range(15),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), - (range(16),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), + ((), '\x90'), + (tuple(range(15)),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), + (tuple(range(16)),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), ({}, '\x80'), (dict([(x,x) for x in range(15)]), '\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'), (dict([(x,x) for x in range(16)]), '\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'), diff --git a/test/test_format.py b/test/test_format.py index 009a764..562ef54 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -10,21 +10,21 @@ def check(src, should): def testSimpleValue(): check("\x93\xc0\xc2\xc3", - [None, False, True]) + (None, False, True,)) def testFixnum(): check("\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", - [[0,64,127], [-32,-16,-1]] + ((0,64,127,), (-32,-16,-1,),) ) def testFixArray(): check("\x92\x90\x91\x91\xc0", - [[],[[None]]], + ((),((None,),),), ) def testFixRaw(): check("\x94\xa0\xa1a\xa2bc\xa3def", - ["", "a", "bc", "def"], + ("", "a", "bc", "def",), ) def testFixMap(): @@ -38,25 +38,26 @@ def testUnsignedInt(): "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" "\xce\xff\xff\xff\xff", - [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,), ) def testSignedInt(): check("\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" "\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" "\xd2\xff\xff\xff\xff", - [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1]) + (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,)) def testRaw(): check("\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" "\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", - ["", "a", "ab", "", "a", "ab"]) + ("", "a", "ab", "", "a", "ab")) def testArray(): check("\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" "\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02" "\xc2\xc3", - [[], [None], [False,True], [], [None], [False,True]]) + ((), (None,), (False,True), (), (None,), (False,True)) + ) def testMap(): check( @@ -67,8 +68,8 @@ def testMap(): "\xdf\x00\x00\x00\x00" "\xdf\x00\x00\x00\x01\xc0\xc2" "\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", - [{}, {None: False}, {True: False, None: False}, {}, - {None: False}, {True: False, None: False}]) + ({}, {None: False}, {True: False, None: False}, {}, + {None: False}, {True: False, None: False})) if __name__ == '__main__': main() diff --git a/test/test_pack.py b/test/test_pack.py index 86badb5..5dec068 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -17,7 +17,7 @@ def testPack(): 1.0, "", "a", "a"*31, "a"*32, None, True, False, - [], [[]], [[], None], + (), ((),), ((), None,), {None: 0}, (1<<23), ] From 232aced926635a7054ac4081d472529cdf96f749 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 18:03:33 +0900 Subject: [PATCH 0340/1172] Update gitignore. --- python/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/.gitignore b/python/.gitignore index 81df440..430c633 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -1,3 +1,5 @@ MANIFEST build/* dist/* +*.pyc +*.pyo From 9d168d2ae872f024f62e7e6653e6a8a17473c33a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 18:03:33 +0900 Subject: [PATCH 0341/1172] Update gitignore. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 81df440..430c633 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ MANIFEST build/* dist/* +*.pyc +*.pyo From c2dd22ec102d12374b97f4ae32dccdaaca8630cd Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 19 Dec 2009 22:09:29 +0900 Subject: [PATCH 0342/1172] c,cpp: add msgpack_vrefbuffer_migrate, msgpack::vrefbuffer::migrate --- c/vrefbuffer.c | 29 ++++++++++++++++++++++++++++- c/vrefbuffer.h | 1 + cpp/vrefbuffer.hpp | 7 +++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/c/vrefbuffer.c b/c/vrefbuffer.c index bbaf61d..2bf97af 100644 --- a/c/vrefbuffer.c +++ b/c/vrefbuffer.c @@ -77,7 +77,7 @@ int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, const char* buf, unsigned int len) { if(vbuf->tail == vbuf->end) { - const size_t nused = vbuf->end - vbuf->array; + const size_t nused = vbuf->tail - vbuf->array; const size_t nnext = nused * 2; struct iovec* nvec = (struct iovec*)realloc( @@ -133,3 +133,30 @@ int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, } } +int msgpack_vrefbuffer_migrate(msgpack_vrefbuffer* vbuf, msgpack_vrefbuffer* to) +{ + const size_t tosize = to->tail - to->array; + if(vbuf->tail + tosize < vbuf->end) { + const size_t nused = vbuf->tail - vbuf->array; + const size_t nsize = vbuf->end - vbuf->array; + const size_t reqsize = nused + tosize; + size_t nnext = nsize * 2; + while(nnext < reqsize) { + nnext *= 2; + } + + struct iovec* nvec = (struct iovec*)realloc( + vbuf->array, sizeof(struct iovec)*nnext); + if(nvec == NULL) { + return -1; + } + + vbuf->array = nvec; + vbuf->end = nvec + nnext; + vbuf->tail = nvec + nused; + } + + memcpy(vbuf->tail, vbuf->array, sizeof(struct iovec)*tosize); + return 0; +} + diff --git a/c/vrefbuffer.h b/c/vrefbuffer.h index 063075f..9f24f14 100644 --- a/c/vrefbuffer.h +++ b/c/vrefbuffer.h @@ -76,6 +76,7 @@ int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, const char* buf, unsigned int len); +int msgpack_vrefbuffer_migrate(msgpack_vrefbuffer* vbuf, msgpack_vrefbuffer* to); int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len) { diff --git a/cpp/vrefbuffer.hpp b/cpp/vrefbuffer.hpp index 549d77f..c8eca7b 100644 --- a/cpp/vrefbuffer.hpp +++ b/cpp/vrefbuffer.hpp @@ -71,6 +71,13 @@ public: return msgpack_vrefbuffer_veclen(this); } + void migrate(vrefbuffer* to) + { + if(msgpack_vrefbuffer_migrate(this, to) < 0) { + throw std::bad_alloc(); + } + } + private: typedef msgpack_vrefbuffer base; From 9817e9b18df85ed8f6afdc24f9f5c2f8228b609d Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 4 Jan 2010 11:59:52 +0900 Subject: [PATCH 0343/1172] Perl: support NVTYPE=="long double" or IVTYPE="long long" environment. --- perl/pack.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index 01dd5b6..589cee8 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -37,10 +37,19 @@ static void need(enc_t *enc, STRLEN len); #include "msgpack/pack_template.h" -#define _PACK_WRAPPER(t) msgpack_pack_##t -#define PACK_WRAPPER(t) _PACK_WRAPPER(t) #define INIT_SIZE 32 /* initial scalar size to be allocated */ +#if IVSIZE == 8 +# define PACK_IV msgpack_pack_int64 +#elif IVSIZE == 4 +# define PACK_IV msgpack_pack_int32 +#elif IVSIZE == 2 +# define PACK_IV msgpack_pack_int16 +#else +# error "msgpack only supports IVSIZE = 8,4,2 environment." +#endif + + static void need(enc_t *enc, STRLEN len) { if (enc->cur + len >= enc->end) { @@ -155,11 +164,12 @@ static void _msgpack_pack_sv(enc_t *enc, SV* sv) { msgpack_pack_raw_body(enc, csv, len); } } else if (SvNOKp(sv)) { - PACK_WRAPPER(NVTYPE)(enc, SvNVX(sv)); + /* XXX long double is not supported yet. */ + msgpack_pack_double(enc, (double)SvNVX(sv)); } else if (SvIOK_UV(sv)) { msgpack_pack_uint32(enc, SvUV(sv)); } else if (SvIOKp(sv)) { - PACK_WRAPPER(IVTYPE)(enc, SvIV(sv)); + PACK_IV(enc, SvIV(sv)); } else if (SvROK(sv)) { _msgpack_pack_rv(enc, SvRV(sv)); } else if (!SvOK(sv)) { From 7873e41e00d3edc9cef567762c1e44a1627cab15 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 4 Jan 2010 12:10:46 +0900 Subject: [PATCH 0344/1172] Perl: change for release Data-MessagePack-0.09 --- perl/Changes | 5 +++++ perl/MANIFEST.SKIP | 2 +- perl/lib/Data/MessagePack.pm | 6 +++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/perl/Changes b/perl/Changes index c6e370c..4ff69c2 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.09 + + - support NVTYPE=="long double" or IVTYPE=="long long" environment + (thanks to Jun Kuriyama++) + 0.08 - fixed PVNV issue... diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP index a318e47..f634035 100644 --- a/perl/MANIFEST.SKIP +++ b/perl/MANIFEST.SKIP @@ -20,6 +20,6 @@ ^\.git/ \.sw[pon]$ ^\.gitignore$ -ppport.h \.o$ \.bs$ +^Data-MessagePack-[0-9.]+/ diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index dde7302..4a95e45 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.08'; +our $VERSION = '0.09'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; @@ -44,6 +44,10 @@ Pack the string as int when the value looks like int(EXPERIMENTAL). Tokuhiro Matsuno +=head1 THANKS TO + +Jun Kuriyama + =head1 LICENSE This library is free software; you can redistribute it and/or modify From d76093b148c1082a1465041263810595f3fd5b3b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 8 Jan 2010 17:34:14 +0900 Subject: [PATCH 0345/1172] import MessagePack for Java --- java-plan1/MessagePack.java | 37 -- java-plan1/Packer.java | 266 ----------- java-plan1/Unpacker.java | 401 ---------------- java-plan1/run.sh | 3 - java-plan1/test.java | 94 ---- java-plan2/build.xml | 15 - java-plan2/src/org/msgpack/GenericArray.java | 25 - .../src/org/msgpack/GenericBoolean.java | 17 - java-plan2/src/org/msgpack/GenericLong.java | 17 - java-plan2/src/org/msgpack/GenericMap.java | 25 - java-plan2/src/org/msgpack/GenericObject.java | 52 --- java-plan2/src/org/msgpack/GenericRaw.java | 59 --- java-plan2/src/org/msgpack/GenericRawRef.java | 55 --- java-plan2/src/org/msgpack/GenericShort.java | 29 -- .../src/org/msgpack/MessageConvertable.java | 7 - .../src/org/msgpack/MessageMergeable.java | 7 - .../src/org/msgpack/MessagePackException.java | 10 - .../src/org/msgpack/MessagePackable.java | 9 - java-plan2/src/org/msgpack/Packer.java | 332 -------------- .../src/org/msgpack/UnbufferedUnpacker.java | 74 --- .../src/org/msgpack/UnpackException.java | 35 -- .../src/org/msgpack/UnpackIterator.java | 53 --- java-plan2/src/org/msgpack/Unpacker.java | 269 ----------- .../src/org/msgpack/impl/ArrayBuilder.java | 7 - .../msgpack/impl/GenericObjectBuilder.java | 127 ----- .../impl/GenericZeroCopyObjectBuilder.java | 12 - .../src/org/msgpack/impl/MapBuilder.java | 8 - .../src/org/msgpack/impl/ObjectBuilder.java | 16 - .../msgpack/impl/SpecificObjectBuilder.java | 203 -------- .../src/org/msgpack/impl/UnpackerImpl.java | 395 ---------------- .../src/org/msgpack/schema/ArraySchema.java | 61 --- .../org/msgpack/schema/ClassGenerator.java | 230 ---------- .../src/org/msgpack/schema/ClassSchema.java | 39 -- .../src/org/msgpack/schema/FieldSchema.java | 31 -- .../msgpack/schema/GenericClassSchema.java | 89 ---- .../msgpack/schema/GenericFieldSchema.java | 25 - .../src/org/msgpack/schema/IntSchema.java | 58 --- .../src/org/msgpack/schema/LongSchema.java | 58 --- .../src/org/msgpack/schema/MapSchema.java | 70 --- .../src/org/msgpack/schema/ObjectSchema.java | 33 -- .../org/msgpack/schema/PrimitiveSchema.java | 21 - .../src/org/msgpack/schema/RawSchema.java | 44 -- .../src/org/msgpack/schema/SSchemaParser.java | 246 ---------- java-plan2/src/org/msgpack/schema/Schema.java | 93 ---- .../src/org/msgpack/schema/ShortSchema.java | 46 -- .../msgpack/schema/SpecificClassSchema.java | 156 ------- .../msgpack/schema/SpecificFieldSchema.java | 78 ---- .../src/org/msgpack/schema/StringSchema.java | 48 -- java-plan2/test/Generate.java | 18 - .../src/serializers/msgpack/MediaContent.java | 277 ----------- .../msgpack/MessagePackSerializer.java | 70 --- .../serializers/msgpack/MediaContent.mpacs | 21 - {java-plan3 => java}/build.xml | 3 + .../src/org/msgpack/MessageMergeable.java | 2 +- .../src/org/msgpack/MessagePackable.java | 2 +- .../src/org/msgpack/MessageTypeException.java | 2 +- .../src/org/msgpack/Packer.java | 2 +- .../src/org/msgpack/Schema.java | 2 +- .../src/org/msgpack/UnbufferedUnpacker.java | 2 +- .../src/org/msgpack/UnpackException.java | 2 +- .../src/org/msgpack/UnpackIterator.java | 2 +- .../src/org/msgpack/Unpacker.java | 2 +- .../src/org/msgpack/impl/UnpackerImpl.java | 17 + .../src/org/msgpack/schema/ArraySchema.java | 2 +- .../src/org/msgpack/schema/ByteSchema.java | 2 +- .../org/msgpack/schema/ClassGenerator.java | 2 +- .../src/org/msgpack/schema/ClassSchema.java | 2 +- .../src/org/msgpack/schema/DoubleSchema.java | 2 +- .../src/org/msgpack/schema/FieldSchema.java | 2 +- .../src/org/msgpack/schema/FloatSchema.java | 2 +- .../msgpack/schema/GenericClassSchema.java | 2 +- .../src/org/msgpack/schema/GenericSchema.java | 2 +- .../src/org/msgpack/schema/IArraySchema.java | 2 +- .../src/org/msgpack/schema/IMapSchema.java | 2 +- .../src/org/msgpack/schema/IntSchema.java | 2 +- .../src/org/msgpack/schema/LongSchema.java | 2 +- .../src/org/msgpack/schema/MapSchema.java | 3 +- .../src/org/msgpack/schema/RawSchema.java | 2 +- .../msgpack/schema/ReflectionClassSchema.java | 0 .../src/org/msgpack/schema/SSchemaParser.java | 2 +- .../src/org/msgpack/schema/ShortSchema.java | 2 +- .../msgpack/schema/SpecificClassSchema.java | 2 +- .../src/org/msgpack/schema/StringSchema.java | 2 +- {java-plan3 => java}/test/Generate.java | 0 java/test/README | 7 + .../tpc/src/serializers/BenchmarkRunner.java | 434 ++++++++++++++++++ .../src/serializers/msgpack/MediaContent.java | 0 .../serializers/msgpack/MediaContent.mpacs | 0 .../msgpack/MessagePackDynamicSerializer.java | 0 .../msgpack/MessagePackGenericSerializer.java | 0 .../MessagePackIndirectSerializer.java | 0 .../MessagePackSpecificSerializer.java | 0 92 files changed, 490 insertions(+), 4499 deletions(-) delete mode 100644 java-plan1/MessagePack.java delete mode 100644 java-plan1/Packer.java delete mode 100644 java-plan1/Unpacker.java delete mode 100755 java-plan1/run.sh delete mode 100644 java-plan1/test.java delete mode 100644 java-plan2/build.xml delete mode 100644 java-plan2/src/org/msgpack/GenericArray.java delete mode 100644 java-plan2/src/org/msgpack/GenericBoolean.java delete mode 100644 java-plan2/src/org/msgpack/GenericLong.java delete mode 100644 java-plan2/src/org/msgpack/GenericMap.java delete mode 100644 java-plan2/src/org/msgpack/GenericObject.java delete mode 100644 java-plan2/src/org/msgpack/GenericRaw.java delete mode 100644 java-plan2/src/org/msgpack/GenericRawRef.java delete mode 100644 java-plan2/src/org/msgpack/GenericShort.java delete mode 100644 java-plan2/src/org/msgpack/MessageConvertable.java delete mode 100644 java-plan2/src/org/msgpack/MessageMergeable.java delete mode 100644 java-plan2/src/org/msgpack/MessagePackException.java delete mode 100644 java-plan2/src/org/msgpack/MessagePackable.java delete mode 100644 java-plan2/src/org/msgpack/Packer.java delete mode 100644 java-plan2/src/org/msgpack/UnbufferedUnpacker.java delete mode 100644 java-plan2/src/org/msgpack/UnpackException.java delete mode 100644 java-plan2/src/org/msgpack/UnpackIterator.java delete mode 100644 java-plan2/src/org/msgpack/Unpacker.java delete mode 100644 java-plan2/src/org/msgpack/impl/ArrayBuilder.java delete mode 100644 java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java delete mode 100644 java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java delete mode 100644 java-plan2/src/org/msgpack/impl/MapBuilder.java delete mode 100644 java-plan2/src/org/msgpack/impl/ObjectBuilder.java delete mode 100644 java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java delete mode 100644 java-plan2/src/org/msgpack/impl/UnpackerImpl.java delete mode 100644 java-plan2/src/org/msgpack/schema/ArraySchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/ClassGenerator.java delete mode 100644 java-plan2/src/org/msgpack/schema/ClassSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/FieldSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/GenericClassSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/GenericFieldSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/IntSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/LongSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/MapSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/ObjectSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/PrimitiveSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/RawSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/SSchemaParser.java delete mode 100644 java-plan2/src/org/msgpack/schema/Schema.java delete mode 100644 java-plan2/src/org/msgpack/schema/ShortSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/SpecificClassSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/StringSchema.java delete mode 100644 java-plan2/test/Generate.java delete mode 100644 java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java delete mode 100644 java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java delete mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs rename {java-plan3 => java}/build.xml (78%) rename {java-plan3 => java}/src/org/msgpack/MessageMergeable.java (94%) rename {java-plan3 => java}/src/org/msgpack/MessagePackable.java (94%) rename {java-plan3 => java}/src/org/msgpack/MessageTypeException.java (96%) rename {java-plan3 => java}/src/org/msgpack/Packer.java (99%) rename {java-plan3 => java}/src/org/msgpack/Schema.java (98%) rename {java-plan3 => java}/src/org/msgpack/UnbufferedUnpacker.java (97%) rename {java-plan3 => java}/src/org/msgpack/UnpackException.java (94%) rename {java-plan3 => java}/src/org/msgpack/UnpackIterator.java (96%) rename {java-plan3 => java}/src/org/msgpack/Unpacker.java (99%) rename {java-plan3 => java}/src/org/msgpack/impl/UnpackerImpl.java (95%) rename {java-plan3 => java}/src/org/msgpack/schema/ArraySchema.java (98%) rename {java-plan3 => java}/src/org/msgpack/schema/ByteSchema.java (97%) rename {java-plan3 => java}/src/org/msgpack/schema/ClassGenerator.java (99%) rename {java-plan3 => java}/src/org/msgpack/schema/ClassSchema.java (98%) rename {java-plan3 => java}/src/org/msgpack/schema/DoubleSchema.java (97%) rename {java-plan3 => java}/src/org/msgpack/schema/FieldSchema.java (95%) rename {java-plan3 => java}/src/org/msgpack/schema/FloatSchema.java (97%) rename {java-plan3 => java}/src/org/msgpack/schema/GenericClassSchema.java (98%) rename {java-plan3 => java}/src/org/msgpack/schema/GenericSchema.java (98%) rename {java-plan3 => java}/src/org/msgpack/schema/IArraySchema.java (94%) rename {java-plan3 => java}/src/org/msgpack/schema/IMapSchema.java (94%) rename {java-plan3 => java}/src/org/msgpack/schema/IntSchema.java (97%) rename {java-plan3 => java}/src/org/msgpack/schema/LongSchema.java (97%) rename {java-plan3 => java}/src/org/msgpack/schema/MapSchema.java (97%) rename {java-plan3 => java}/src/org/msgpack/schema/RawSchema.java (98%) rename {java-plan3 => java}/src/org/msgpack/schema/ReflectionClassSchema.java (100%) rename {java-plan3 => java}/src/org/msgpack/schema/SSchemaParser.java (99%) rename {java-plan3 => java}/src/org/msgpack/schema/ShortSchema.java (97%) rename {java-plan3 => java}/src/org/msgpack/schema/SpecificClassSchema.java (98%) rename {java-plan3 => java}/src/org/msgpack/schema/StringSchema.java (98%) rename {java-plan3 => java}/test/Generate.java (100%) create mode 100644 java/test/README create mode 100644 java/test/thrift-protobuf-compare/tpc/src/serializers/BenchmarkRunner.java rename {java-plan3 => java}/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java (100%) rename {java-plan2 => java}/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs (100%) rename {java-plan3 => java}/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java (100%) rename {java-plan3 => java}/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java (100%) rename {java-plan3 => java}/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java (100%) rename {java-plan3 => java}/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java (100%) diff --git a/java-plan1/MessagePack.java b/java-plan1/MessagePack.java deleted file mode 100644 index 0a4f442..0000000 --- a/java-plan1/MessagePack.java +++ /dev/null @@ -1,37 +0,0 @@ -import java.nio.ByteBuffer; -import java.io.InputStream; -import java.io.IOException; - -public class MessagePack { - - static public Object unpack(InputStream source) - { - // FIXME not implemented yet - return null; - } - - static public Object unpack(byte[] source, int len) throws IOException - { - // FIXME not implemented yet - return null; - } - - static public Object unpack(ByteBuffer source) throws IOException - { - // FIXME not implemented yet - return null; - } - - static public Object unpack(ByteBuffer[] source) throws IOException - { - // FIXME not implemented yet - return null; - } -} - -/* -public interface MessagePackable { - public void toMessagePack(Packer pk); -} -*/ - diff --git a/java-plan1/Packer.java b/java-plan1/Packer.java deleted file mode 100644 index ea6b2d4..0000000 --- a/java-plan1/Packer.java +++ /dev/null @@ -1,266 +0,0 @@ -import java.io.*; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.math.BigInteger; - -public class Packer { - protected byte[] castBytes = new byte[9]; - protected ByteBuffer castBuffer = ByteBuffer.wrap(castBytes); - protected OutputStream out; - - public Packer(OutputStream out) - { - this.out = out; - } - - public Packer packByte(byte d) throws IOException - { - if(d < -(1<<5)) { - castBytes[0] = (byte)0xd1; - castBytes[1] = d; - out.write(castBytes, 0, 2); - } else { - out.write(d); - } - return this; - } - - public Packer packShort(short d) throws IOException - { - if(d < -(1<<5)) { - if(d < -(1<<7)) { - // signed 16 - castBytes[0] = (byte)0xd1; - castBuffer.putShort(1, d); - out.write(castBytes, 0, 3); - } else { - // signed 8 - castBytes[0] = (byte)0xd0; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } - } else if(d < (1<<7)) { - // fixnum - out.write((byte)d); - } else { - if(d < (1<<8)) { - // unsigned 8 - castBytes[0] = (byte)0xcc; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } else { - // unsigned 16 - castBytes[0] = (byte)0xcd; - castBuffer.putShort(1, d); - out.write(castBytes, 0, 3); - } - } - return this; - } - - public Packer packInt(int d) throws IOException - { - if(d < -(1<<5)) { - if(d < -(1<<15)) { - // signed 32 - castBytes[0] = (byte)0xd2; - castBuffer.putInt(1, d); - out.write(castBytes, 0, 5); - } else if(d < -(1<<7)) { - // signed 16 - castBytes[0] = (byte)0xd1; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - } else { - // signed 8 - castBytes[0] = (byte)0xd0; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } - } else if(d < (1<<7)) { - // fixnum - out.write((byte)d); - } else { - if(d < (1<<8)) { - // unsigned 8 - castBytes[0] = (byte)0xcc; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } else if(d < (1<<16)) { - // unsigned 16 - castBytes[0] = (byte)0xcd; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - } else { - // unsigned 32 - castBytes[0] = (byte)0xce; - castBuffer.putInt(1, d); - out.write(castBytes, 0, 5); - } - } - return this; - } - - public Packer packLong(long d) throws IOException - { - if(d < -(1L<<5)) { - if(d < -(1L<<15)) { - if(d < -(1L<<31)) { - // signed 64 - castBytes[0] = (byte)0xd3; - castBuffer.putLong(1, d); - out.write(castBytes, 0, 9); - } else { - // signed 32 - castBytes[0] = (byte)0xd2; - castBuffer.putInt(1, (int)d); - out.write(castBytes, 0, 5); - } - } else { - if(d < -(1<<7)) { - // signed 16 - castBytes[0] = (byte)0xd1; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - } else { - // signed 8 - castBytes[0] = (byte)0xd0; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } - } - } else if(d < (1<<7)) { - // fixnum - out.write((byte)d); - } else { - if(d < (1L<<16)) { - if(d < (1<<8)) { - // unsigned 8 - castBytes[0] = (byte)0xcc; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } else { - // unsigned 16 - castBytes[0] = (byte)0xcd; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - //System.out.println("pack uint 16 "+(short)d); - } - } else { - if(d < (1L<<32)) { - // unsigned 32 - castBytes[0] = (byte)0xce; - castBuffer.putInt(1, (int)d); - out.write(castBytes, 0, 5); - } else { - // unsigned 64 - castBytes[0] = (byte)0xcf; - castBuffer.putLong(1, d); - out.write(castBytes, 0, 9); - } - } - } - return this; - } - - public Packer packFloat(float d) throws IOException - { - castBytes[0] = (byte)0xca; - castBuffer.putFloat(1, d); - out.write(castBytes, 0, 5); - return this; - } - - public Packer packDouble(double d) throws IOException - { - castBytes[0] = (byte)0xcb; - castBuffer.putDouble(1, d); - out.write(castBytes, 0, 9); - return this; - } - - public Packer packNil() throws IOException - { - out.write((byte)0xc0); - return this; - } - - public Packer packTrue() throws IOException - { - out.write((byte)0xc3); - return this; - } - - public Packer packFalse() throws IOException - { - out.write((byte)0xc2); - return this; - } - - public Packer packArray(int n) throws IOException - { - if(n < 16) { - final int d = 0x90 | n; - out.write((byte)d); - } else if(n < 65536) { - castBytes[0] = (byte)0xdc; - castBuffer.putShort(1, (short)n); - out.write(castBytes, 0, 3); - } else { - castBytes[0] = (byte)0xdd; - castBuffer.putInt(1, n); - out.write(castBytes, 0, 5); - } - return this; - } - - public Packer packMap(int n) throws IOException - { - if(n < 16) { - final int d = 0x80 | n; - out.write((byte)d); - } else if(n < 65536) { - castBytes[0] = (byte)0xde; - castBuffer.putShort(1, (short)n); - out.write(castBytes, 0, 3); - } else { - castBytes[0] = (byte)0xdf; - castBuffer.putInt(1, n); - out.write(castBytes, 0, 5); - } - return this; - } - - public Packer packRaw(int n) throws IOException - { - if(n < 32) { - final int d = 0xa0 | n; - out.write((byte)d); - } else if(n < 65536) { - castBytes[0] = (byte)0xda; - castBuffer.putShort(1, (short)n); - out.write(castBytes, 0, 3); - } else { - castBytes[0] = (byte)0xdb; - castBuffer.putInt(1, n); - out.write(castBytes, 0, 5); - } - return this; - } - - public Packer packRawBody(byte[] b) throws IOException - { - out.write(b); - return this; - } - - public Packer packRawBody(byte[] b, int off, int length) throws IOException - { - out.write(b, off, length); - return this; - } - - //public Packer pack(Object o) throws IOException -} - diff --git a/java-plan1/Unpacker.java b/java-plan1/Unpacker.java deleted file mode 100644 index 32945de..0000000 --- a/java-plan1/Unpacker.java +++ /dev/null @@ -1,401 +0,0 @@ -import java.io.*; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.math.BigInteger; - -public class Unpacker { - protected static final int CS_HEADER = 0x00; - protected static final int CS_FLOAT = 0x0a; - protected static final int CS_DOUBLE = 0x0b; - protected static final int CS_UINT_8 = 0x0c; - protected static final int CS_UINT_16 = 0x0d; - protected static final int CS_UINT_32 = 0x0e; - protected static final int CS_UINT_64 = 0x0f; - protected static final int CS_INT_8 = 0x10; - protected static final int CS_INT_16 = 0x11; - protected static final int CS_INT_32 = 0x12; - protected static final int CS_INT_64 = 0x13; - protected static final int CS_RAW_16 = 0x1a; - protected static final int CS_RAW_32 = 0x1b; - protected static final int CS_ARRAY_16 = 0x1c; - protected static final int CS_ARRAY_32 = 0x1d; - protected static final int CS_MAP_16 = 0x1e; - protected static final int CS_MAP_32 = 0x1f; - protected static final int ACS_RAW_VALUE = 0x20; - protected static final int CT_ARRAY_ITEM = 0x00; - protected static final int CT_MAP_KEY = 0x01; - protected static final int CT_MAP_VALUE = 0x02; - - protected static final int MAX_STACK_SIZE = 16; - - protected int cs = CS_HEADER; - protected int trail = 0; - protected int top = -1; - protected boolean finished = false; - protected int[] stack_ct = new int[MAX_STACK_SIZE]; - protected int[] stack_count = new int[MAX_STACK_SIZE]; - protected Object[] stack_obj = new Object[MAX_STACK_SIZE]; - protected Object[] stack_map_key = new Object[MAX_STACK_SIZE]; - //protected Object user; - protected ByteBuffer castBuffer = ByteBuffer.allocate(8); - - public Object getData() - { - return stack_obj[0]; - } - - public boolean isFinished() - { - return finished; - } - - public void reset() - { - for(int i=0; i <= top; ++top) { - stack_ct[top] = 0; - stack_count[top] = 0; - stack_obj[top] = null; - stack_map_key[top] = null; - } - cs = CS_HEADER; - trail = 0; - top = -1; - finished = false; - } - - @SuppressWarnings("unchecked") - public int execute(byte[] src, int off, int length) throws IOException - { - if(off >= length) { return off; } - - int limit = length; - int i = off; - int count; - - Object obj = null; - - _out: do { - _header_again: { - //System.out.println("while i:"+i+" limit:"+limit); - - int b = src[i]; - - _push: { - _fixed_trail_again: - if(cs == CS_HEADER) { - - if((b & 0x80) == 0) { // Positive Fixnum - //System.out.println("positive fixnum "+b); - obj = b; - break _push; - } - - if((b & 0xe0) == 0xe0) { // Negative Fixnum - //System.out.println("negative fixnum "+b); - obj = b; - break _push; - } - - if((b & 0xe0) == 0xa0) { // FixRaw - trail = b & 0x1f; - if(trail == 0) { - obj = new byte[0]; - break _push; - } - cs = ACS_RAW_VALUE; - break _fixed_trail_again; - } - - if((b & 0xf0) == 0x90) { // FixArray - if(top >= MAX_STACK_SIZE) { - throw new IOException("parse error"); - } - count = b & 0x0f; - ++top; - stack_obj[top] = new ArrayList(count); - stack_ct[top] = CT_ARRAY_ITEM; - stack_count[top] = count; - //System.out.println("fixarray count:"+count); - break _header_again; - } - - if((b & 0xf0) == 0x80) { // FixMap - if(top >= MAX_STACK_SIZE) { - throw new IOException("parse error"); - } - count = b & 0x0f; - ++top; - stack_obj[top] = new HashMap(count); - stack_ct[top] = CT_MAP_KEY; - stack_count[top] = count; - //System.out.println("fixmap count:"+count); - break _header_again; - } - - switch(b & 0xff) { // FIXME - case 0xc0: // nil - obj = null; - break _push; - case 0xc2: // false - obj = false; - break _push; - case 0xc3: // true - obj = true; - break _push; - case 0xca: // float - case 0xcb: // double - case 0xcc: // unsigned int 8 - case 0xcd: // unsigned int 16 - case 0xce: // unsigned int 32 - case 0xcf: // unsigned int 64 - case 0xd0: // signed int 8 - case 0xd1: // signed int 16 - case 0xd2: // signed int 32 - case 0xd3: // signed int 64 - trail = 1 << (b & 0x03); - cs = b & 0x1f; - //System.out.println("a trail "+trail+" cs:"+cs); - break _fixed_trail_again; - case 0xda: // raw 16 - case 0xdb: // raw 32 - case 0xdc: // array 16 - case 0xdd: // array 32 - case 0xde: // map 16 - case 0xdf: // map 32 - trail = 2 << (b & 0x01); - cs = b & 0x1f; - //System.out.println("b trail "+trail+" cs:"+cs); - break _fixed_trail_again; - default: - //System.out.println("unknown b "+(b&0xff)); - throw new IOException("parse error"); - } - - } // _fixed_trail_again - - do { - _fixed_trail_again: { - - if(limit - i <= trail) { break _out; } - int n = i + 1; - i += trail; - - switch(cs) { - case CS_FLOAT: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - obj = castBuffer.getFloat(0); - //System.out.println("float "+obj); - break _push; - case CS_DOUBLE: - castBuffer.rewind(); - castBuffer.put(src, n, 8); - obj = castBuffer.getDouble(0); - //System.out.println("double "+obj); - break _push; - case CS_UINT_8: - //System.out.println(n); - //System.out.println(src[n]); - //System.out.println(src[n+1]); - //System.out.println(src[n-1]); - obj = ((int)src[n]) & 0xff; - //System.out.println("uint8 "+obj); - break _push; - case CS_UINT_16: - //System.out.println(src[n]); - //System.out.println(src[n+1]); - castBuffer.rewind(); - castBuffer.put(src, n, 2); - obj = ((int)castBuffer.getShort(0)) & 0xffff; - //System.out.println("uint 16 "+obj); - break _push; - case CS_UINT_32: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - { - // FIXME always long? - int o = castBuffer.getInt(0); - if(o < 0) { - obj = ((long)o) & 0xffffffffL; - } else { - obj = o; - } - } - //System.out.println("uint 32 "+obj); - break _push; - case CS_UINT_64: - castBuffer.rewind(); - castBuffer.put(src, n, 8); - { - // FIXME always BigInteger? - long o = castBuffer.getLong(0); - if(o < 0) { - obj = BigInteger.valueOf(o & 0x7fffffffL).setBit(31); - } else { - obj = o; - } - } - throw new IOException("uint 64 is not supported"); - case CS_INT_8: - obj = (int)src[n]; - break _push; - case CS_INT_16: - castBuffer.rewind(); - castBuffer.put(src, n, 2); - obj = (int)castBuffer.getShort(0); - break _push; - case CS_INT_32: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - obj = (int)castBuffer.getInt(0); - break _push; - case CS_INT_64: - castBuffer.rewind(); - castBuffer.put(src, n, 8); - { - // FIXME always long? - long o = castBuffer.getLong(0); - if(o <= 0x7fffffffL && o > -0x80000000L) { - obj = (int)o; - } else { - obj = o; - } - } - //System.out.println("long "+obj); - break _push; - case CS_RAW_16: - castBuffer.rewind(); - castBuffer.put(src, n, 2); - trail = ((int)castBuffer.getShort(0)) & 0xffff; - if(trail == 0) { - obj = new byte[0]; - break _push; - } - cs = ACS_RAW_VALUE; - break _fixed_trail_again; - case CS_RAW_32: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - // FIXME overflow check - trail = castBuffer.getInt(0) & 0x7fffffff; - if(trail == 0) { - obj = new byte[0]; - break _push; - } - cs = ACS_RAW_VALUE; - case ACS_RAW_VALUE: - obj = ByteBuffer.wrap(src, n, trail); // FIXME プール? コピー - break _push; - case CS_ARRAY_16: - if(top >= MAX_STACK_SIZE) { - throw new IOException("parse error"); - } - castBuffer.rewind(); - castBuffer.put(src, n, 2); - count = ((int)castBuffer.getShort(0)) & 0xffff; - ++top; - stack_obj[top] = new ArrayList(count); - stack_ct[top] = CT_ARRAY_ITEM; - stack_count[top] = count; - break _header_again; - case CS_ARRAY_32: - if(top >= MAX_STACK_SIZE) { - throw new IOException("parse error"); - } - castBuffer.rewind(); - castBuffer.put(src, n, 4); - // FIXME overflow check - count = castBuffer.getInt(0) & 0x7fffffff; - ++top; - stack_obj[top] = new ArrayList(count); - stack_ct[top] = CT_ARRAY_ITEM; - stack_count[top] = count; - break _header_again; - case CS_MAP_16: - if(top >= MAX_STACK_SIZE) { - throw new IOException("parse error"); - } - castBuffer.rewind(); - castBuffer.put(src, n, 2); - count = ((int)castBuffer.getShort(0)) & 0xffff; - ++top; - stack_obj[top] = new HashMap(count); - stack_ct[top] = CT_MAP_KEY; - stack_count[top] = count; - //System.out.println("fixmap count:"+count); - break _header_again; - case CS_MAP_32: - if(top >= MAX_STACK_SIZE) { - throw new IOException("parse error"); - } - castBuffer.rewind(); - castBuffer.put(src, n, 4); - // FIXME overflow check - count = castBuffer.getInt(0) & 0x7fffffff; - ++top; - stack_obj[top] = new HashMap(count); - stack_ct[top] = CT_MAP_KEY; - stack_count[top] = count; - //System.out.println("fixmap count:"+count); - break _header_again; - default: - throw new IOException("parse error"); - } - - } // _fixed_trail_again - } while(true); - } // _push - - do { - _push: { - //System.out.println("push top:"+top); - if(top == -1) { - stack_obj[0] = obj; - finished = true; - break _out; - } - - switch(stack_ct[top]) { - case CT_ARRAY_ITEM: - //System.out.println("array item "+obj); - ((ArrayList)(stack_obj[top])).add(obj); - if(--stack_count[top] == 0) { - obj = stack_obj[top]; - stack_obj[top] = null; - --top; - break _push; - } - break _header_again; - case CT_MAP_KEY: - //System.out.println("map key:"+top+" "+obj); - stack_map_key[top] = obj; - stack_ct[top] = CT_MAP_VALUE; - break _header_again; - case CT_MAP_VALUE: - //System.out.println("map value:"+top+" "+obj); - ((HashMap)(stack_obj[top])).put(stack_map_key[top], obj); - if(--stack_count[top] == 0) { - obj = stack_obj[top]; - stack_map_key[top] = null; - stack_obj[top] = null; - --top; - break _push; - } - stack_ct[top] = CT_MAP_KEY; - break _header_again; - default: - throw new IOException("parse error"); - } - } // _push - } while(true); - - } // _header_again - cs = CS_HEADER; - ++i; - } while(i < limit); // _out - - return i; - } -} - diff --git a/java-plan1/run.sh b/java-plan1/run.sh deleted file mode 100755 index 1539a37..0000000 --- a/java-plan1/run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -javac MessagePack.java Packer.java Unpacker.java test.java -java test diff --git a/java-plan1/test.java b/java-plan1/test.java deleted file mode 100644 index 5b2349f..0000000 --- a/java-plan1/test.java +++ /dev/null @@ -1,94 +0,0 @@ -import java.util.*; -import java.io.*; - -class OpenByteArrayOutputStream extends ByteArrayOutputStream { - int getCount() { return count; } - byte[] getBuffer() { return buf; } -} - - -public class test { - - public static void main(String[] args) throws IOException - { - testSimple(); - testStreaming(); - } - - - public static void testSimple() throws IOException - { - OpenByteArrayOutputStream out = new OpenByteArrayOutputStream(); - - Packer pk = new Packer(out); - pk.packArray(3) - .packInt(1) - .packByte((byte)2) - .packDouble(0.3); - - Unpacker pac = new Unpacker(); - int nlen = pac.execute(out.getBuffer(), 0, out.getCount()); - - if(pac.isFinished()) { - System.out.println("testSimple: "+pac.getData()); - } - } - - - public static void testStreaming() throws IOException - { - OpenByteArrayOutputStream out = new OpenByteArrayOutputStream(); - - //// - // sender - // - // initialize the streaming serializer - Packer pk = new Packer(out); - - // serialize 2 objects - pk.packArray(3) - .packInt(0) - .packByte((byte)1) - .packDouble(0.2); - pk.packArray(3) - .packInt(3) - .packByte((byte)4) - .packDouble(0.5); - - // send it through the network - InputStream sock = new ByteArrayInputStream(out.getBuffer(), 0, out.getCount()); - - - //// - // receiver - // - // initialize the streaming deserializer - Unpacker pac = new Unpacker(); - int parsed = 0; - - byte[] buf = new byte[1024]; - int buflen = 0; - - while(true) { - // receive data from the network - int c = sock.read(); - if(c < 0) { return; } - - buf[buflen++] = (byte)c; - - // deserialize - parsed = pac.execute(buf, parsed, buflen); - if(pac.isFinished()) { - // get an object - Object msg = pac.getData(); - System.out.println("testStreaming: "+msg); - - // reset the streaming deserializer - pac.reset(); - buflen = 0; - parsed = 0; - } - } - } -} - diff --git a/java-plan2/build.xml b/java-plan2/build.xml deleted file mode 100644 index d382ce0..0000000 --- a/java-plan2/build.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/java-plan2/src/org/msgpack/GenericArray.java b/java-plan2/src/org/msgpack/GenericArray.java deleted file mode 100644 index 53799ca..0000000 --- a/java-plan2/src/org/msgpack/GenericArray.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.msgpack; - -import java.util.List; -import java.util.ArrayList; - -public class GenericArray extends GenericObject { - private ArrayList array; - - public GenericArray(int length) - { - this.array = new ArrayList(length); - } - - public void add(GenericObject element) - { - array.add(element); - } - - @Override - public List asArray() - { - return array; - } -} - diff --git a/java-plan2/src/org/msgpack/GenericBoolean.java b/java-plan2/src/org/msgpack/GenericBoolean.java deleted file mode 100644 index 908128f..0000000 --- a/java-plan2/src/org/msgpack/GenericBoolean.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.msgpack; - -public class GenericBoolean extends GenericObject { - boolean value; - - public GenericBoolean(boolean value) - { - this.value = value; - } - - @Override - public boolean asBoolean() - { - return value; - } -} - diff --git a/java-plan2/src/org/msgpack/GenericLong.java b/java-plan2/src/org/msgpack/GenericLong.java deleted file mode 100644 index 7cc9110..0000000 --- a/java-plan2/src/org/msgpack/GenericLong.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.msgpack; - -public class GenericLong extends GenericObject { - long value; - - public GenericLong(long value) - { - this.value = value; - } - - @Override - public long asLong() - { - return value; - } -} - diff --git a/java-plan2/src/org/msgpack/GenericMap.java b/java-plan2/src/org/msgpack/GenericMap.java deleted file mode 100644 index 0a5512a..0000000 --- a/java-plan2/src/org/msgpack/GenericMap.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.msgpack; - -import java.util.Map; -import java.util.HashMap; - -public class GenericMap extends GenericObject { - private HashMap map; - - public GenericMap(int length) - { - this.map = new HashMap(length); - } - - public void put(GenericObject key, GenericObject value) - { - map.put(key, value); - } - - @Override - public Map asMap() - { - return map; - } -} - diff --git a/java-plan2/src/org/msgpack/GenericObject.java b/java-plan2/src/org/msgpack/GenericObject.java deleted file mode 100644 index 32eacd3..0000000 --- a/java-plan2/src/org/msgpack/GenericObject.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.msgpack; - -import java.util.List; -import java.util.Map; - -public abstract class GenericObject { - public boolean asBoolean() - { - throw new RuntimeException("type error"); - } - - public byte asByte() - { - throw new RuntimeException("type error"); - } - - public short asShort() - { - throw new RuntimeException("type error"); - } - - public int asInt() - { - throw new RuntimeException("type error"); - } - - public long asLong() - { - throw new RuntimeException("type error"); - } - - public String asString() - { - throw new RuntimeException("type error"); - } - - public byte[] asBytes() - { - throw new RuntimeException("type error"); - } - - public List asArray() - { - throw new RuntimeException("type error"); - } - - public Map asMap() - { - throw new RuntimeException("type error"); - } -} - diff --git a/java-plan2/src/org/msgpack/GenericRaw.java b/java-plan2/src/org/msgpack/GenericRaw.java deleted file mode 100644 index 6de03b2..0000000 --- a/java-plan2/src/org/msgpack/GenericRaw.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.msgpack; - -import java.nio.charset.Charset; - -public class GenericRaw extends GenericObject { - byte[] bytes; - String string; - - public GenericRaw() - { - this.bytes = new byte[0]; - this.string = null; - } - - public GenericRaw(byte[] bytes) - { - this.bytes = bytes; - this.string = null; - } - - public GenericRaw(String string) - { - this.bytes = null; - this.string = string; - } - - public synchronized void setString(String string) - { - this.string = string; - this.bytes = null; - } - - public synchronized void setBytes(byte[] bytes) - { - this.bytes = bytes; - this.string = null; - } - - private static Charset UTF8_CHARSET = Charset.forName("UTF-8"); - - @Override - public synchronized String asString() - { - if(string == null) { - return string = new String(bytes, UTF8_CHARSET); - } - return string; - } - - @Override - public synchronized byte[] asBytes() - { - if(bytes == null) { - return bytes = string.getBytes(UTF8_CHARSET); - } - return bytes; - } -} - diff --git a/java-plan2/src/org/msgpack/GenericRawRef.java b/java-plan2/src/org/msgpack/GenericRawRef.java deleted file mode 100644 index 7007810..0000000 --- a/java-plan2/src/org/msgpack/GenericRawRef.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.msgpack; - -import java.nio.charset.Charset; - -public class GenericRawRef extends GenericRaw { - int offset; - int length; - - public GenericRawRef(byte[] bytes, int offset, int length) - { - this.bytes = bytes; - this.offset = offset; - this.length = length; - this.string = null; - } - - public GenericRawRef(String string) - { - this.bytes = null; - this.string = string; - } - - public synchronized void setString(String string) - { - this.string = string; - this.bytes = null; - } - - public synchronized void setBytes(byte[] bytes) - { - this.bytes = bytes; - this.string = null; - } - - private static Charset UTF8_CHARSET = Charset.forName("UTF-8"); - - @Override - public synchronized String asString() - { - if(string == null) { - return string = new String(bytes, UTF8_CHARSET); - } - return string; - } - - @Override - public synchronized byte[] asBytes() - { - if(bytes == null) { - return bytes = string.getBytes(UTF8_CHARSET); - } - return bytes; - } -} - diff --git a/java-plan2/src/org/msgpack/GenericShort.java b/java-plan2/src/org/msgpack/GenericShort.java deleted file mode 100644 index eb2463e..0000000 --- a/java-plan2/src/org/msgpack/GenericShort.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.msgpack; - -public class GenericShort extends GenericObject { - short value; - - public GenericShort(short value) - { - this.value = value; - } - - @Override - public short asShort() - { - return value; - } - - @Override - public int asInt() - { - return value; - } - - @Override - public long asLong() - { - return value; - } -} - diff --git a/java-plan2/src/org/msgpack/MessageConvertable.java b/java-plan2/src/org/msgpack/MessageConvertable.java deleted file mode 100644 index 68c123e..0000000 --- a/java-plan2/src/org/msgpack/MessageConvertable.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.msgpack; - -public interface MessageConvertable -{ - public void messageConvert(GenericObject obj); -} - diff --git a/java-plan2/src/org/msgpack/MessageMergeable.java b/java-plan2/src/org/msgpack/MessageMergeable.java deleted file mode 100644 index dc50749..0000000 --- a/java-plan2/src/org/msgpack/MessageMergeable.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.msgpack; - -public interface MessageMergeable { - public void setField(int index, Object value); - public Object getField(int index); -} - diff --git a/java-plan2/src/org/msgpack/MessagePackException.java b/java-plan2/src/org/msgpack/MessagePackException.java deleted file mode 100644 index dc37989..0000000 --- a/java-plan2/src/org/msgpack/MessagePackException.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.msgpack; - -public class MessagePackException extends RuntimeException -{ - public MessagePackException(String message) - { - super(message); - } -} - diff --git a/java-plan2/src/org/msgpack/MessagePackable.java b/java-plan2/src/org/msgpack/MessagePackable.java deleted file mode 100644 index 1efff3f..0000000 --- a/java-plan2/src/org/msgpack/MessagePackable.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.msgpack; - -import java.io.IOException; - -public interface MessagePackable -{ - public void messagePack(Packer pk) throws IOException; -} - diff --git a/java-plan2/src/org/msgpack/Packer.java b/java-plan2/src/org/msgpack/Packer.java deleted file mode 100644 index 4545849..0000000 --- a/java-plan2/src/org/msgpack/Packer.java +++ /dev/null @@ -1,332 +0,0 @@ -package org.msgpack; - -import java.io.OutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.util.List; -import java.util.Map; - -public class Packer { - protected byte[] castBytes = new byte[9]; - protected ByteBuffer castBuffer = ByteBuffer.wrap(castBytes); - protected OutputStream out; - - public Packer(OutputStream out) - { - this.out = out; - } - - public Packer packByte(byte d) throws IOException - { - if(d < -(1<<5)) { - castBytes[0] = (byte)0xd1; - castBytes[1] = d; - out.write(castBytes, 0, 2); - } else { - out.write(d); - } - return this; - } - - public Packer packShort(short d) throws IOException - { - if(d < -(1<<5)) { - if(d < -(1<<7)) { - // signed 16 - castBytes[0] = (byte)0xd1; - castBuffer.putShort(1, d); - out.write(castBytes, 0, 3); - } else { - // signed 8 - castBytes[0] = (byte)0xd0; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } - } else if(d < (1<<7)) { - // fixnum - out.write((byte)d); - } else { - if(d < (1<<8)) { - // unsigned 8 - castBytes[0] = (byte)0xcc; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } else { - // unsigned 16 - castBytes[0] = (byte)0xcd; - castBuffer.putShort(1, d); - out.write(castBytes, 0, 3); - } - } - return this; - } - - public Packer packInt(int d) throws IOException - { - if(d < -(1<<5)) { - if(d < -(1<<15)) { - // signed 32 - castBytes[0] = (byte)0xd2; - castBuffer.putInt(1, d); - out.write(castBytes, 0, 5); - } else if(d < -(1<<7)) { - // signed 16 - castBytes[0] = (byte)0xd1; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - } else { - // signed 8 - castBytes[0] = (byte)0xd0; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } - } else if(d < (1<<7)) { - // fixnum - out.write((byte)d); - } else { - if(d < (1<<8)) { - // unsigned 8 - castBytes[0] = (byte)0xcc; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } else if(d < (1<<16)) { - // unsigned 16 - castBytes[0] = (byte)0xcd; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - } else { - // unsigned 32 - castBytes[0] = (byte)0xce; - castBuffer.putInt(1, d); - out.write(castBytes, 0, 5); - } - } - return this; - } - - public Packer packLong(long d) throws IOException - { - if(d < -(1L<<5)) { - if(d < -(1L<<15)) { - if(d < -(1L<<31)) { - // signed 64 - castBytes[0] = (byte)0xd3; - castBuffer.putLong(1, d); - out.write(castBytes, 0, 9); - } else { - // signed 32 - castBytes[0] = (byte)0xd2; - castBuffer.putInt(1, (int)d); - out.write(castBytes, 0, 5); - } - } else { - if(d < -(1<<7)) { - // signed 16 - castBytes[0] = (byte)0xd1; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - } else { - // signed 8 - castBytes[0] = (byte)0xd0; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } - } - } else if(d < (1<<7)) { - // fixnum - out.write((byte)d); - } else { - if(d < (1L<<16)) { - if(d < (1<<8)) { - // unsigned 8 - castBytes[0] = (byte)0xcc; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } else { - // unsigned 16 - castBytes[0] = (byte)0xcd; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - //System.out.println("pack uint 16 "+(short)d); - } - } else { - if(d < (1L<<32)) { - // unsigned 32 - castBytes[0] = (byte)0xce; - castBuffer.putInt(1, (int)d); - out.write(castBytes, 0, 5); - } else { - // unsigned 64 - castBytes[0] = (byte)0xcf; - castBuffer.putLong(1, d); - out.write(castBytes, 0, 9); - } - } - } - return this; - } - - public Packer packFloat(float d) throws IOException - { - castBytes[0] = (byte)0xca; - castBuffer.putFloat(1, d); - out.write(castBytes, 0, 5); - return this; - } - - public Packer packDouble(double d) throws IOException - { - castBytes[0] = (byte)0xcb; - castBuffer.putDouble(1, d); - out.write(castBytes, 0, 9); - return this; - } - - public Packer packNil() throws IOException - { - out.write((byte)0xc0); - return this; - } - - public Packer packTrue() throws IOException - { - out.write((byte)0xc3); - return this; - } - - public Packer packFalse() throws IOException - { - out.write((byte)0xc2); - return this; - } - - public Packer packArray(int n) throws IOException - { - if(n < 16) { - final int d = 0x90 | n; - out.write((byte)d); - } else if(n < 65536) { - castBytes[0] = (byte)0xdc; - castBuffer.putShort(1, (short)n); - out.write(castBytes, 0, 3); - } else { - castBytes[0] = (byte)0xdd; - castBuffer.putInt(1, n); - out.write(castBytes, 0, 5); - } - return this; - } - - public Packer packMap(int n) throws IOException - { - if(n < 16) { - final int d = 0x80 | n; - out.write((byte)d); - } else if(n < 65536) { - castBytes[0] = (byte)0xde; - castBuffer.putShort(1, (short)n); - out.write(castBytes, 0, 3); - } else { - castBytes[0] = (byte)0xdf; - castBuffer.putInt(1, n); - out.write(castBytes, 0, 5); - } - return this; - } - - public Packer packRaw(int n) throws IOException - { - if(n < 32) { - final int d = 0xa0 | n; - out.write((byte)d); - } else if(n < 65536) { - castBytes[0] = (byte)0xda; - castBuffer.putShort(1, (short)n); - out.write(castBytes, 0, 3); - } else { - castBytes[0] = (byte)0xdb; - castBuffer.putInt(1, n); - out.write(castBytes, 0, 5); - } - return this; - } - - public Packer packRawBody(byte[] b) throws IOException - { - out.write(b); - return this; - } - - public Packer packRawBody(byte[] b, int off, int length) throws IOException - { - out.write(b, off, length); - return this; - } - - private static Charset UTF8_CHARSET = Charset.forName("UTF-8"); - - public Packer packString(String s) throws IOException - { - byte[] b = ((String)s).getBytes(UTF8_CHARSET); - packRaw(b.length); - packRawBody(b); - return this; - } - - public Packer pack(MessagePackable o) throws IOException - { - o.messagePack(this); - return this; - } - - public Packer pack(Object o) throws IOException - { - if(o == null) { - packNil(); - } else if(o instanceof String) { - byte[] b = ((String)o).getBytes(UTF8_CHARSET); - packRaw(b.length); - packRawBody(b); - } else if(o instanceof byte[]) { - byte[] b = (byte[])o; - packRaw(b.length); - packRawBody(b); - } else if(o instanceof List) { - List l = (List)o; - packArray(l.size()); - for(Object i : l) { pack(i); } - } else if(o instanceof Map) { - Map m = (Map)o; - packMap(m.size()); - for(Map.Entry e : m.entrySet()) { - pack(e.getKey()); - pack(e.getValue()); - } - } else if(o instanceof Boolean) { - if((Boolean)o) { - packTrue(); - } else { - packFalse(); - } - } else if(o instanceof Integer) { - packInt((Integer)o); - } else if(o instanceof Long) { - packLong((Long)o); - } else if(o instanceof Short) { - packShort((Short)o); - } else if(o instanceof Byte) { - packByte((Byte)o); - } else if(o instanceof Float) { - packFloat((Float)o); - } else if(o instanceof Double) { - packDouble((Double)o); - } else if(o instanceof MessagePackable) { - ((MessagePackable)o).messagePack(this); - } else { - throw new IOException("unknown object "+o+" ("+o.getClass()+")"); - } - return this; - } -} - diff --git a/java-plan2/src/org/msgpack/UnbufferedUnpacker.java b/java-plan2/src/org/msgpack/UnbufferedUnpacker.java deleted file mode 100644 index 5b5eb97..0000000 --- a/java-plan2/src/org/msgpack/UnbufferedUnpacker.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.msgpack; - -import java.lang.Iterable; -import java.io.InputStream; -import java.io.IOException; -import java.util.Iterator; -import org.msgpack.impl.*; -import org.msgpack.schema.Schema; - -public class UnbufferedUnpacker extends UnpackerImpl { - private int offset; - private boolean finished; - private Object data; - - public UnbufferedUnpacker() - { - super(new GenericObjectBuilder()); - this.offset = 0; - this.finished = false; - } - - public UnbufferedUnpacker useSchema(Schema s) - { - super.setBuilder(new SpecificObjectBuilder(s)); - return this; - } - - public Object getData() - { - return data; - } - - public boolean isFinished() - { - return finished; - } - - public void reset() - { - super.reset(); - this.offset = 0; - } - - int getOffset() - { - return offset; - } - - void setOffset(int offset) - { - this.offset = offset; - } - - public int execute(byte[] buffer) throws UnpackException - { - return execute(buffer, 0, buffer.length); - } - - // FIXME - public int execute(byte[] buffer, int offset, int length) throws UnpackException - { - int noffset = super.execute(buffer, offset + this.offset, length); - this.offset = noffset - offset; - if(super.isFinished()) { - this.data = super.getData(); - this.finished = true; - super.reset(); - } else { - this.finished = false; - } - return noffset; - } -} - diff --git a/java-plan2/src/org/msgpack/UnpackException.java b/java-plan2/src/org/msgpack/UnpackException.java deleted file mode 100644 index 2081faf..0000000 --- a/java-plan2/src/org/msgpack/UnpackException.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.msgpack; - -public class UnpackException extends MessagePackException -{ - public static final int EXTRA_BYTES = 1; - public static final int INSUFFICIENT_BYTES = 0; - public static final int PARSE_ERROR = -1; - - private int errorCode; - private Object data; - - public UnpackException(String message, int errorCode) - { - super(message); - this.errorCode = errorCode; - } - - public UnpackException(String message, int errorCode, Object data) - { - super(message); - this.errorCode = errorCode; - this.data = data; - } - - public int getErrorCode() - { - return errorCode; - } - - public Object getData() - { - return data; - } -} - diff --git a/java-plan2/src/org/msgpack/UnpackIterator.java b/java-plan2/src/org/msgpack/UnpackIterator.java deleted file mode 100644 index 9010101..0000000 --- a/java-plan2/src/org/msgpack/UnpackIterator.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.msgpack; - -import java.io.IOException; -import java.util.Iterator; -import java.util.NoSuchElementException; - -public class UnpackIterator implements Iterator { - private Unpacker pac; - private boolean have; - private Object data; - - UnpackIterator(Unpacker pac) - { - this.pac = pac; - this.have = false; - } - - public boolean hasNext() - { - if(have) { return true; } - try { - while(true) { - if(pac.execute()) { - data = pac.getData(); - pac.reset(); - have = true; - return true; - } - - if(!pac.fill()) { - return false; - } - } - } catch (IOException e) { - return false; - } - } - - public Object next() - { - if(!have) { - throw new NoSuchElementException(); - } - have = false; - return data; - } - - public void remove() - { - throw new UnsupportedOperationException(); - } -} - diff --git a/java-plan2/src/org/msgpack/Unpacker.java b/java-plan2/src/org/msgpack/Unpacker.java deleted file mode 100644 index 5a58ce8..0000000 --- a/java-plan2/src/org/msgpack/Unpacker.java +++ /dev/null @@ -1,269 +0,0 @@ -package org.msgpack; - -import java.lang.Iterable; -import java.io.InputStream; -import java.io.IOException; -import java.util.Iterator; -import org.msgpack.impl.*; -import org.msgpack.schema.Schema; - -public class Unpacker extends UnpackerImpl implements Iterable { - - public static final int DEFAULT_BUFFER_SIZE = 32*1024; - - private int used; - private int offset; - private int parsed; - private byte[] buffer; - private int bufferReserveSize; - private InputStream stream; - - public Unpacker() - { - super(new GenericZeroCopyObjectBuilder()); - this.used = 0; - this.offset = 0; - this.parsed = 0; - this.buffer = new byte[DEFAULT_BUFFER_SIZE]; - this.bufferReserveSize = DEFAULT_BUFFER_SIZE/2; - this.stream = null; - } - - public Unpacker(int bufferReserveSize) - { - super(new GenericZeroCopyObjectBuilder()); - this.used = 0; - this.offset = 0; - this.parsed = 0; - this.buffer = new byte[bufferReserveSize]; - this.bufferReserveSize = bufferReserveSize/2; - this.stream = null; - } - - public Unpacker(InputStream stream) - { - super(new GenericZeroCopyObjectBuilder()); - this.used = 0; - this.offset = 0; - this.parsed = 0; - this.buffer = new byte[DEFAULT_BUFFER_SIZE]; - this.bufferReserveSize = DEFAULT_BUFFER_SIZE/2; - this.stream = stream; - } - - public Unpacker(InputStream stream, int bufferReserveSize) - { - super(new GenericZeroCopyObjectBuilder()); - this.used = 0; - this.offset = 0; - this.parsed = 0; - this.buffer = new byte[bufferReserveSize]; - this.bufferReserveSize = bufferReserveSize/2; - this.stream = stream; - } - - public Unpacker useSchema(Schema s) - { - super.setBuilder(new SpecificObjectBuilder(s)); - return this; - } - - public void reserveBuffer(int size) - { - if(buffer.length - used >= size) { - return; - } - /* - if(used == parsed && buffer.length >= size) { - // rewind buffer - used = 0; - offset = 0; - return; - } - */ - - int nextSize = buffer.length * 2; - while(nextSize < size + used) { - nextSize *= 2; - } - - byte[] tmp = new byte[nextSize]; - System.arraycopy(buffer, offset, tmp, 0, used - offset); - - buffer = tmp; - used -= offset; - offset = 0; - } - - public byte[] getBuffer() - { - return buffer; - } - - public int getBufferOffset() - { - return used; - } - - public int getBufferCapacity() - { - return buffer.length - used; - } - - public void bufferConsumed(int size) - { - used += size; - } - - public void feed(byte[] buffer) - { - feed(buffer, 0, buffer.length); - } - - public void feed(byte[] buffer, int offset, int length) - { - reserveBuffer(length); - System.arraycopy(buffer, offset, this.buffer, this.offset, length); - bufferConsumed(length); - } - - public boolean fill() throws IOException - { - if(stream == null) { - return false; - } - reserveBuffer(bufferReserveSize); - int rl = stream.read(getBuffer(), getBufferOffset(), getBufferCapacity()); - if(rl <= 0) { - return false; - } - bufferConsumed(rl); - return true; - } - - public Iterator iterator() - { - return new UnpackIterator(this); - } - - public boolean execute() throws UnpackException - { - int noffset = super.execute(buffer, offset, used); - if(noffset <= offset) { - return false; - } - parsed += noffset - offset; - offset = noffset; - return super.isFinished(); - } - - public Object getData() - { - return super.getData(); - } - - public void reset() - { - super.reset(); - parsed = 0; - } - - public int getMessageSize() - { - return parsed - offset + used; - } - - public int getParsedSize() - { - return parsed; - } - - public int getNonParsedSize() - { - return used - offset; - } - - public void skipNonparsedBuffer(int size) - { - offset += size; - } - - public void removeNonparsedBuffer() - { - used = offset; - } - - /* - public static class Context { - private boolean finished; - private Object data; - private int offset; - private UnpackerImpl impl; - - public Context() - { - this.finished = false; - this.impl = new UnpackerImpl(); - } - - public boolean isFinished() - { - return finished; - } - - public Object getData() - { - return data; - } - - int getOffset() - { - return offset; - } - - void setFinished(boolean finished) - { - this.finished = finished; - } - - void setData(Object data) - { - this.data = data; - } - - void setOffset(int offset) - { - this.offset = offset; - } - - UnpackerImpl getImpl() - { - return impl; - } - } - - public static int unpack(Context ctx, byte[] buffer) throws UnpackException - { - return unpack(ctx, buffer, 0, buffer.length); - } - - public static int unpack(Context ctx, byte[] buffer, int offset, int length) throws UnpackException - { - UnpackerImpl impl = ctx.getImpl(); - int noffset = impl.execute(buffer, offset + ctx.getOffset(), length); - ctx.setOffset(noffset - offset); - if(impl.isFinished()) { - ctx.setData(impl.getData()); - ctx.setFinished(false); - impl.reset(); - } else { - ctx.setData(null); - ctx.setFinished(true); - } - int parsed = noffset - offset; - ctx.setOffset(parsed); - return noffset; - } - */ -} - diff --git a/java-plan2/src/org/msgpack/impl/ArrayBuilder.java b/java-plan2/src/org/msgpack/impl/ArrayBuilder.java deleted file mode 100644 index 9bb099b..0000000 --- a/java-plan2/src/org/msgpack/impl/ArrayBuilder.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.msgpack.impl; - -public interface ArrayBuilder { - public void add(Object element); - public Object finish(); -} - diff --git a/java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java b/java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java deleted file mode 100644 index 814e302..0000000 --- a/java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java +++ /dev/null @@ -1,127 +0,0 @@ -package org.msgpack.impl; - -import org.msgpack.*; - -public class GenericObjectBuilder implements ObjectBuilder { - public Object createNil() - { - return null; - } - - @Override - public Object createBoolean(boolean v) - { - return new GenericBoolean(v); - } - - @Override - public Object createByte(byte v) - { - //return new GenericByte(v); - return null; // FIXME - } - - @Override - public Object createShort(short v) - { - return new GenericShort(v); - } - - @Override - public Object createInt(int v) - { - //return new GenericInt(v); - return null; // FIXME - } - - @Override - public Object createLong(long v) - { - return new GenericLong(v); - } - - @Override - public Object createFloat(float v) - { - //return new GenericFloat(v); - return null; // FIXME - } - - @Override - public Object createDouble(double v) - { - //return new GenericDouble(v); - return null; // FIXME - } - - @Override - public Object createRaw(byte[] b, int offset, int length) - { - byte[] copy = new byte[length]; - System.arraycopy(b, offset, copy, 0, length); - return new GenericRaw(copy); - } - - @Override - public ArrayBuilder createArray(int length) - { - return new GenericArrayBuilder(length); - } - - @Override - public MapBuilder createMap(int length) - { - return new GenericMapBuilder(length); - } -} - -final class GenericArrayBuilder implements ArrayBuilder { - private GenericArray a; - - GenericArrayBuilder(int length) - { - this.a = new GenericArray(length); - } - - @Override - public void add(Object element) - { - a.add((GenericObject)element); - } - - @Override - public Object finish() - { - return a; - } -} - -final class GenericMapBuilder implements MapBuilder { - private GenericMap m; - private GenericObject key; - - GenericMapBuilder(int length) - { - this.m = new GenericMap(length); - } - - @Override - public void putKey(Object key) - { - this.key = (GenericObject)key; - } - - @Override - public void putValue(Object value) - { - m.put(this.key, (GenericObject)value); - this.key = null; - } - - @Override - public Object finish() - { - return m; - } -} - diff --git a/java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java b/java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java deleted file mode 100644 index 5569fbf..0000000 --- a/java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.msgpack.impl; - -import org.msgpack.*; - -public class GenericZeroCopyObjectBuilder extends GenericObjectBuilder { - @Override - public Object createRaw(byte[] b, int offset, int length) - { - return new GenericRawRef(b, offset, length); - } -} - diff --git a/java-plan2/src/org/msgpack/impl/MapBuilder.java b/java-plan2/src/org/msgpack/impl/MapBuilder.java deleted file mode 100644 index 57859a6..0000000 --- a/java-plan2/src/org/msgpack/impl/MapBuilder.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.msgpack.impl; - -public interface MapBuilder { - public void putKey(Object key); - public void putValue(Object value); - public Object finish(); -} - diff --git a/java-plan2/src/org/msgpack/impl/ObjectBuilder.java b/java-plan2/src/org/msgpack/impl/ObjectBuilder.java deleted file mode 100644 index 3268903..0000000 --- a/java-plan2/src/org/msgpack/impl/ObjectBuilder.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.msgpack.impl; - -public interface ObjectBuilder { - public Object createNil(); - public Object createBoolean(boolean v); - public Object createByte(byte v); - public Object createShort(short v); - public Object createInt(int v); - public Object createLong(long v); - public Object createFloat(float v); - public Object createDouble(double v); - public Object createRaw(byte[] b, int offset, int length); - public ArrayBuilder createArray(int length); - public MapBuilder createMap(int length); -} - diff --git a/java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java b/java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java deleted file mode 100644 index 7748844..0000000 --- a/java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java +++ /dev/null @@ -1,203 +0,0 @@ -package org.msgpack.impl; - -import java.util.List; -import java.util.ArrayList; -import java.util.HashMap; -import org.msgpack.schema.*; - -public final class SpecificObjectBuilder implements ObjectBuilder { - private int top; - private Schema schema; - - public SpecificObjectBuilder(Schema schema) - { - this.top = 0; - this.schema = schema; - } - - void setSchema(Schema s) - { - schema = s; - } - - Schema swapSchema(Schema s) - { - Schema old = schema; - schema = s; - return old; - } - - @Override - public Object createNil() - { - return schema.createNil(); - } - - @Override - public Object createBoolean(boolean v) - { - return schema.createBoolean(v); - } - - @Override - public Object createByte(byte v) - { - return schema.createByte(v); - } - - @Override - public Object createShort(short v) - { - return schema.createShort(v); - } - - @Override - public Object createInt(int v) - { - return schema.createInt(v); - } - - @Override - public Object createLong(long v) - { - return schema.createLong(v); - } - - @Override - public Object createFloat(float v) - { - return schema.createFloat(v); - } - - @Override - public Object createDouble(double v) - { - return schema.createDouble(v); - } - - @Override - public Object createRaw(byte[] b, int offset, int length) - { - return schema.createRaw(b, offset, length); - } - - @Override - public ArrayBuilder createArray(int length) - { - if(schema instanceof ClassSchema) { - return new ClassBuilder(length, (ClassSchema)schema, this); - } - ArraySchema as = (ArraySchema)schema; - return new SpecificArrayBuilder(length, as.getElementType(), this); - } - - @Override - public MapBuilder createMap(int length) - { - MapSchema ms = (MapSchema)schema; - return new SpecificMapBuilder(length, ms.getKeyType(), ms.getValueType(), this); - } -} - -final class SpecificArrayBuilder implements ArrayBuilder { - private ArrayList a; - private SpecificObjectBuilder builder; - private Schema parentSchema; - - SpecificArrayBuilder(int length, Schema elementSchema, SpecificObjectBuilder builder) - { - this.a = new ArrayList(length); - this.builder = builder; - this.parentSchema = builder.swapSchema(elementSchema); - } - - public void add(Object element) - { - a.add(element); - } - - public Object finish() - { - builder.swapSchema(parentSchema); - return a; - } -} - -final class SpecificMapBuilder implements MapBuilder { - private HashMap m; - private Object key; - private Schema keySchema; - private Schema valueSchema; - private SpecificObjectBuilder builder; - private Schema parentSchema; - - SpecificMapBuilder(int length, Schema keySchema, Schema valueSchema, SpecificObjectBuilder builder) - { - this.m = new HashMap(length); - this.keySchema = keySchema; - this.valueSchema = valueSchema; - this.builder = builder; - this.parentSchema = builder.swapSchema(keySchema); - } - - @Override - public void putKey(Object key) - { - this.key = key; - this.builder.setSchema(valueSchema); - } - - @Override - public void putValue(Object value) - { - m.put(this.key, value); - this.key = null; - this.builder.setSchema(keySchema); - } - - @Override - public Object finish() - { - builder.swapSchema(parentSchema); - return m; - } -} - -final class ClassBuilder implements ArrayBuilder { - private Object object; - private int index; - private List fields; - private SpecificObjectBuilder builder; - private Schema parentSchema; - - ClassBuilder(int length, ClassSchema schema, SpecificObjectBuilder builder) - { - this.object = schema.newInstance(); - this.index = 0; - this.fields = schema.getFields(); - this.builder = builder; - this.parentSchema = builder.swapSchema(fields.get(0).getType()); - // FIXME check length - } - - @Override - public void add(Object element) - { - FieldSchema f = fields.get(index++); // FIXME check fields.size - f.setFieldValue(object, element); // XXX FIXME debug - if(fields.size() > index) { - builder.setSchema( fields.get(index).getType() ); - } else { - builder.setSchema( null ); - // FIXME: builder.setSchema(new InvalidFieldSchema); - } - } - - @Override - public Object finish() - { - builder.swapSchema(parentSchema); - return object; - } -} - diff --git a/java-plan2/src/org/msgpack/impl/UnpackerImpl.java b/java-plan2/src/org/msgpack/impl/UnpackerImpl.java deleted file mode 100644 index 9f88072..0000000 --- a/java-plan2/src/org/msgpack/impl/UnpackerImpl.java +++ /dev/null @@ -1,395 +0,0 @@ -package org.msgpack.impl; - -import java.nio.ByteBuffer; -//import java.math.BigInteger; -import org.msgpack.UnpackException; - -public class UnpackerImpl { - static final int CS_HEADER = 0x00; - static final int CS_FLOAT = 0x0a; - static final int CS_DOUBLE = 0x0b; - static final int CS_UINT_8 = 0x0c; - static final int CS_UINT_16 = 0x0d; - static final int CS_UINT_32 = 0x0e; - static final int CS_UINT_64 = 0x0f; - static final int CS_INT_8 = 0x10; - static final int CS_INT_16 = 0x11; - static final int CS_INT_32 = 0x12; - static final int CS_INT_64 = 0x13; - static final int CS_RAW_16 = 0x1a; - static final int CS_RAW_32 = 0x1b; - static final int CS_ARRAY_16 = 0x1c; - static final int CS_ARRAY_32 = 0x1d; - static final int CS_MAP_16 = 0x1e; - static final int CS_MAP_32 = 0x1f; - static final int ACS_RAW_VALUE = 0x20; - static final int CT_ARRAY_ITEM = 0x00; - static final int CT_MAP_KEY = 0x01; - static final int CT_MAP_VALUE = 0x02; - - static final int MAX_STACK_SIZE = 16; - - protected int cs = CS_HEADER; - protected int trail = 0; - protected int top = -1; - protected boolean finished = false; - protected Object data = null; - protected int[] stack_ct = new int[MAX_STACK_SIZE]; - protected int[] stack_count = new int[MAX_STACK_SIZE]; - protected Object[] stack_obj = new Object[MAX_STACK_SIZE]; - protected ByteBuffer castBuffer = ByteBuffer.allocate(8); - protected ObjectBuilder builder; - - protected UnpackerImpl(ObjectBuilder builder) - { - this.builder = builder; - } - - protected void setBuilder(ObjectBuilder builder) - { - this.builder = builder; - } - - protected Object getData() - { - return data; - } - - protected boolean isFinished() - { - return finished; - } - - protected void reset() - { - for(int i=0; i <= top; ++top) { - stack_ct[top] = 0; - stack_count[top] = 0; - stack_obj[top] = null; - } - cs = CS_HEADER; - trail = 0; - top = -1; - finished = false; - data = null; - } - - @SuppressWarnings("unchecked") - protected int execute(byte[] src, int off, int length) throws UnpackException - { - if(off >= length) { return off; } - - int limit = length; - int i = off; - int count; - - Object obj = null; - - _out: do { - _header_again: { - //System.out.println("while i:"+i+" limit:"+limit); - - int b = src[i]; - - _push: { - _fixed_trail_again: - if(cs == CS_HEADER) { - - if((b & 0x80) == 0) { // Positive Fixnum - //System.out.println("positive fixnum "+b); - obj = builder.createByte((byte)b); - break _push; - } - - if((b & 0xe0) == 0xe0) { // Negative Fixnum - //System.out.println("negative fixnum "+b); - obj = builder.createByte((byte)b); - break _push; - } - - if((b & 0xe0) == 0xa0) { // FixRaw - trail = b & 0x1f; - if(trail == 0) { - obj = builder.createRaw(new byte[0], 0, 0); - break _push; - } - cs = ACS_RAW_VALUE; - break _fixed_trail_again; - } - - if((b & 0xf0) == 0x90) { // FixArray - if(top >= MAX_STACK_SIZE) { - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - count = b & 0x0f; - ++top; - stack_obj[top] = builder.createArray(count); - stack_ct[top] = CT_ARRAY_ITEM; - stack_count[top] = count; - //System.out.println("fixarray count:"+count); - break _header_again; - } - - if((b & 0xf0) == 0x80) { // FixMap - if(top >= MAX_STACK_SIZE) { - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - count = b & 0x0f; - ++top; - stack_obj[top] = builder.createMap(count); - stack_ct[top] = CT_MAP_KEY; - stack_count[top] = count; - //System.out.println("fixmap count:"+count); - break _header_again; - } - - switch(b & 0xff) { // FIXME - case 0xc0: // nil - obj = builder.createNil(); - break _push; - case 0xc2: // false - obj = builder.createBoolean(false); - break _push; - case 0xc3: // true - obj = builder.createBoolean(true); - break _push; - case 0xca: // float - case 0xcb: // double - case 0xcc: // unsigned int 8 - case 0xcd: // unsigned int 16 - case 0xce: // unsigned int 32 - case 0xcf: // unsigned int 64 - case 0xd0: // signed int 8 - case 0xd1: // signed int 16 - case 0xd2: // signed int 32 - case 0xd3: // signed int 64 - trail = 1 << (b & 0x03); - cs = b & 0x1f; - //System.out.println("a trail "+trail+" cs:"+cs); - break _fixed_trail_again; - case 0xda: // raw 16 - case 0xdb: // raw 32 - case 0xdc: // array 16 - case 0xdd: // array 32 - case 0xde: // map 16 - case 0xdf: // map 32 - trail = 2 << (b & 0x01); - cs = b & 0x1f; - //System.out.println("b trail "+trail+" cs:"+cs); - break _fixed_trail_again; - default: - //System.out.println("unknown b "+(b&0xff)); - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - - } // _fixed_trail_again - - do { - _fixed_trail_again: { - - if(limit - i <= trail) { break _out; } - int n = i + 1; - i += trail; - - switch(cs) { - case CS_FLOAT: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - obj = builder.createFloat( castBuffer.getFloat(0) ); - //System.out.println("float "+obj); - break _push; - case CS_DOUBLE: - castBuffer.rewind(); - castBuffer.put(src, n, 8); - obj = builder.createDouble( castBuffer.getDouble(0) ); - //System.out.println("double "+obj); - break _push; - case CS_UINT_8: - //System.out.println(n); - //System.out.println(src[n]); - //System.out.println(src[n+1]); - //System.out.println(src[n-1]); - obj = builder.createShort( (short)((src[n]) & 0xff) ); - //System.out.println("uint8 "+obj); - break _push; - case CS_UINT_16: - //System.out.println(src[n]); - //System.out.println(src[n+1]); - castBuffer.rewind(); - castBuffer.put(src, n, 2); - obj = builder.createInt( ((int)castBuffer.getShort(0)) & 0xffff ); - //System.out.println("uint 16 "+obj); - break _push; - case CS_UINT_32: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - obj = builder.createLong( ((long)castBuffer.getInt(0)) & 0xffffffffL ); - //System.out.println("uint 32 "+obj); - break _push; - case CS_UINT_64: - castBuffer.rewind(); - castBuffer.put(src, n, 8); - { - long o = castBuffer.getLong(0); - if(o < 0) { - // FIXME - //obj = GenericBigInteger.valueOf(o & 0x7fffffffL).setBit(31); - } else { - obj = builder.createLong( o ); - } - } - throw new UnpackException("uint 64 bigger than 0x7fffffff is not supported", UnpackException.PARSE_ERROR); - case CS_INT_8: - obj = builder.createByte( src[n] ); - break _push; - case CS_INT_16: - castBuffer.rewind(); - castBuffer.put(src, n, 2); - obj = builder.createShort( castBuffer.getShort(0) ); - break _push; - case CS_INT_32: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - obj = builder.createInt( castBuffer.getInt(0) ); - break _push; - case CS_INT_64: - castBuffer.rewind(); - castBuffer.put(src, n, 8); - obj = builder.createLong( castBuffer.getLong(0) ); - break _push; - case CS_RAW_16: - castBuffer.rewind(); - castBuffer.put(src, n, 2); - trail = ((int)castBuffer.getShort(0)) & 0xffff; - if(trail == 0) { - obj = builder.createRaw(new byte[0], 0, 0); - break _push; - } - cs = ACS_RAW_VALUE; - break _fixed_trail_again; - case CS_RAW_32: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - // FIXME overflow check - trail = castBuffer.getInt(0) & 0x7fffffff; - if(trail == 0) { - obj = builder.createRaw(new byte[0], 0, 0); - break _push; - } - cs = ACS_RAW_VALUE; - case ACS_RAW_VALUE: - obj = builder.createRaw(src, n, trail); - break _push; - case CS_ARRAY_16: - if(top >= MAX_STACK_SIZE) { - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - castBuffer.rewind(); - castBuffer.put(src, n, 2); - count = ((int)castBuffer.getShort(0)) & 0xffff; - ++top; - stack_obj[top] = builder.createArray(count); - stack_ct[top] = CT_ARRAY_ITEM; - stack_count[top] = count; - break _header_again; - case CS_ARRAY_32: - if(top >= MAX_STACK_SIZE) { - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - castBuffer.rewind(); - castBuffer.put(src, n, 4); - // FIXME overflow check - count = castBuffer.getInt(0) & 0x7fffffff; - ++top; - stack_obj[top] = builder.createArray(count); - stack_ct[top] = CT_ARRAY_ITEM; - stack_count[top] = count; - break _header_again; - case CS_MAP_16: - if(top >= MAX_STACK_SIZE) { - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - castBuffer.rewind(); - castBuffer.put(src, n, 2); - count = ((int)castBuffer.getShort(0)) & 0xffff; - ++top; - stack_obj[top] = builder.createMap(count); - stack_ct[top] = CT_MAP_KEY; - stack_count[top] = count; - //System.out.println("fixmap count:"+count); - break _header_again; - case CS_MAP_32: - if(top >= MAX_STACK_SIZE) { - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - castBuffer.rewind(); - castBuffer.put(src, n, 4); - // FIXME overflow check - count = castBuffer.getInt(0) & 0x7fffffff; - ++top; - stack_obj[top] = builder.createMap(count); - stack_ct[top] = CT_MAP_KEY; - stack_count[top] = count; - //System.out.println("fixmap count:"+count); - break _header_again; - default: - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - - } // _fixed_trail_again - } while(true); - } // _push - - do { - _push: { - //System.out.println("push top:"+top); - if(top == -1) { - ++i; - data = obj; - finished = true; - break _out; - } - - switch(stack_ct[top]) { - case CT_ARRAY_ITEM: - //System.out.println("array item "+obj); - ((ArrayBuilder)stack_obj[top]).add(obj); - if(--stack_count[top] == 0) { - obj = ((ArrayBuilder)stack_obj[top]).finish(); - stack_obj[top] = null; - --top; - break _push; - } - break _header_again; - case CT_MAP_KEY: - //System.out.println("map key:"+top+" "+obj); - MapBuilder mb = (MapBuilder)stack_obj[top]; - mb.putKey(obj); - stack_ct[top] = CT_MAP_VALUE; - break _header_again; - case CT_MAP_VALUE: - //System.out.println("map value:"+top+" "+obj); - ((MapBuilder)stack_obj[top]).putValue(obj); - if(--stack_count[top] == 0) { - obj = ((MapBuilder)stack_obj[top]).finish(); - stack_obj[top] = null; - --top; - break _push; - } - stack_ct[top] = CT_MAP_KEY; - break _header_again; - default: - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - } // _push - } while(true); - - } // _header_again - cs = CS_HEADER; - ++i; - } while(i < limit); // _out - - return i; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/ArraySchema.java b/java-plan2/src/org/msgpack/schema/ArraySchema.java deleted file mode 100644 index 4b05190..0000000 --- a/java-plan2/src/org/msgpack/schema/ArraySchema.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.msgpack.schema; - -import java.util.List; -import java.util.ArrayList; -import java.io.IOException; -import org.msgpack.*; - -public class ArraySchema extends Schema { - private Schema elementType; - - public ArraySchema(Schema elementType) - { - super("array"); - this.elementType = elementType; - } - - public Schema getElementType() - { - return elementType; - } - - @Override - public String getFullName() - { - return "ArrayList<"+elementType.getFullName()+">"; - } - - @Override - public String getExpression() - { - return "(array "+elementType.getExpression()+")"; - } - - @Override - @SuppressWarnings("unchecked") - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - List d = (List)obj; - pk.packArray(d.size()); - for(Object e : d) { - elementType.pack(pk, e); - } - } - - @Override - @SuppressWarnings("unchecked") - public Object convert(GenericObject obj) - { - List d = obj.asArray(); - List g = new ArrayList(); - for(GenericObject o : d) { - g.add( elementType.convert(o) ); - } - return g; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/ClassGenerator.java b/java-plan2/src/org/msgpack/schema/ClassGenerator.java deleted file mode 100644 index 25a9620..0000000 --- a/java-plan2/src/org/msgpack/schema/ClassGenerator.java +++ /dev/null @@ -1,230 +0,0 @@ -package org.msgpack.schema; - -import java.util.ArrayList; -import java.util.List; -import java.io.IOException; -import java.io.File; -import java.io.FileOutputStream; -import java.io.Writer; - -public class ClassGenerator { - private ClassSchema schema; - private Writer writer; - private int indent; - - private ClassGenerator(Writer writer) - { - this.writer = writer; - this.indent = 0; - } - - public static void write(Schema schema, Writer dest) throws IOException - { - if(!(schema instanceof ClassSchema)) { - throw new RuntimeException("schema is not class schema"); - } - ClassSchema cs = (ClassSchema)schema; - new ClassGenerator(dest).run(cs); - } - - private void run(ClassSchema cs) throws IOException - { - List subclasses = new ArrayList(); - for(FieldSchema f : cs.getFields()) { - findSubclassSchema(subclasses, f.getType()); - } - - for(ClassSchema sub : subclasses) { - sub.setNamespace(cs.getNamespace()); - sub.setImports(cs.getImports()); - } - - this.schema = cs; - - writeHeader(); - - writeClass(); - - for(ClassSchema sub : subclasses) { - this.schema = sub; - writeSubclass(); - } - - writeFooter(); - - this.schema = null; - writer.flush(); - } - - private void findSubclassSchema(List dst, Schema s) - { - if(s instanceof ClassSchema) { - ClassSchema cs = (ClassSchema)s; - if(!dst.contains(cs)) { dst.add(cs); } - for(FieldSchema f : cs.getFields()) { - findSubclassSchema(dst, f.getType()); - } - } else if(s instanceof ArraySchema) { - ArraySchema as = (ArraySchema)s; - findSubclassSchema(dst, as.getElementType()); - } else if(s instanceof MapSchema) { - MapSchema as = (MapSchema)s; - findSubclassSchema(dst, as.getKeyType()); - findSubclassSchema(dst, as.getValueType()); - } - } - - private void writeHeader() throws IOException - { - if(schema.getNamespace() != null) { - line("package "+schema.getNamespace()+";"); - line(); - } - line("import java.util.*;"); - line("import java.io.*;"); - line("import org.msgpack.*;"); - line("import org.msgpack.schema.*;"); - } - - private void writeFooter() throws IOException - { - } - - private void writeClass() throws IOException - { - line(); - line("public final class "+schema.getName()+" implements MessagePackable, MessageConvertable"); - line("{"); - pushIndent(); - writeSchema(); - writeMemberVariables(); - writeMemberFunctions(); - popIndent(); - line("}"); - } - - private void writeSubclass() throws IOException - { - line(); - line("final class "+schema.getName()+" implements MessagePackable, MessageConvertable"); - line("{"); - pushIndent(); - writeSchema(); - writeMemberVariables(); - writeMemberFunctions(); - popIndent(); - line("}"); - } - - private void writeSchema() throws IOException - { - line("private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load(\""+schema.getExpression()+"\");"); - line("public static ClassSchema getSchema() { return _SCHEMA; }"); - } - - private void writeMemberVariables() throws IOException - { - line(); - for(FieldSchema f : schema.getFields()) { - line("public "+f.getType().getFullName()+" "+f.getName()+";"); - } - } - - private void writeMemberFunctions() throws IOException - { - // void messagePack(Packer pk) - // boolean equals(Object obj) - // int hashCode() - // void set(int _index, Object _value) - // Object get(int _index); - // getXxx() - // setXxx(Xxx xxx) - writeConstructors(); - writeAccessors(); - writePackFunction(); - writeConvertFunction(); - } - - private void writeConstructors() throws IOException - { - line(); - line("public "+schema.getName()+"() { }"); - } - - private void writeAccessors() throws IOException - { - // FIXME - //line(); - //for(FieldSchema f : schema.getFields()) { - // line(""); - //} - } - - private void writePackFunction() throws IOException - { - line(); - line("@Override"); - line("public void messagePack(Packer pk) throws IOException"); - line("{"); - pushIndent(); - line("List _f = _SCHEMA.getFields();"); - line("pk.packArray("+schema.getFields().size()+");"); - int i = 0; - for(FieldSchema f : schema.getFields()) { - line("_f.get("+i+").getType().pack(pk, "+f.getName()+");"); - ++i; - } - popIndent(); - line("}"); - } - - private void writeConvertFunction() throws IOException - { - line(); - line("@Override"); - line("@SuppressWarnings(\"unchecked\")"); - line("public void messageConvert(GenericObject obj)"); - line("{"); - pushIndent(); - line("List _l = obj.asArray();"); - line("List _f = _SCHEMA.getFields();"); - int i = 0; - for(FieldSchema f : schema.getFields()) { - line("if(_l.size() <= "+i+") { return; } "+f.getName()+" = ("+f.getType().getFullName()+")_f.get("+i+").getType().convert(_l.get("+i+"));"); - ++i; - } - popIndent(); - line("}"); - line(); - line("public static "+schema.getName()+" convert(GenericObject obj)"); - line("{"); - pushIndent(); - line("return ("+schema.getName()+")_SCHEMA.convert(obj);"); - popIndent(); - line("}"); - } - - private void line(String str) throws IOException - { - for(int i=0; i < indent; ++i) { - writer.write("\t"); - } - writer.write(str+"\n"); - } - - private void line() throws IOException - { - writer.write("\n"); - } - - private void pushIndent() - { - indent += 1; - } - - private void popIndent() - { - indent -= 1; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/ClassSchema.java b/java-plan2/src/org/msgpack/schema/ClassSchema.java deleted file mode 100644 index 3343fca..0000000 --- a/java-plan2/src/org/msgpack/schema/ClassSchema.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.msgpack.schema; - -import java.util.List; - -public abstract class ClassSchema extends Schema { - protected String namespace; - protected List imports; - - public ClassSchema(String name, String namespace, List imports) - { - super(name); - this.namespace = namespace; - this.imports = imports; - } - - public String getNamespace() - { - return namespace; - } - - public List getImports() - { - return imports; - } - - void setNamespace(String namespace) - { - this.namespace = namespace; - } - - void setImports(List imports) - { - this.imports = imports; - } - - public abstract List getFields(); - public abstract Object newInstance(); -} - diff --git a/java-plan2/src/org/msgpack/schema/FieldSchema.java b/java-plan2/src/org/msgpack/schema/FieldSchema.java deleted file mode 100644 index 31a132c..0000000 --- a/java-plan2/src/org/msgpack/schema/FieldSchema.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.msgpack.schema; - -public abstract class FieldSchema { - private String name; - private Schema type; - - public FieldSchema(String name, Schema type) - { - this.name = name; - this.type = type; - } - - public String getName() - { - return this.name; - } - - public Schema getType() - { - return type; - } - - public String getExpression() - { - return "(field "+name+" "+type.getExpression()+")"; - } - - public abstract Object getFieldValue(Object obj); - public abstract void setFieldValue(Object obj, Object value); -} - diff --git a/java-plan2/src/org/msgpack/schema/GenericClassSchema.java b/java-plan2/src/org/msgpack/schema/GenericClassSchema.java deleted file mode 100644 index f1e2b44..0000000 --- a/java-plan2/src/org/msgpack/schema/GenericClassSchema.java +++ /dev/null @@ -1,89 +0,0 @@ -package org.msgpack.schema; - -import java.util.List; -import java.util.Map; -import java.util.HashMap; -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import org.msgpack.*; - - -public class GenericClassSchema extends ClassSchema { - private List fields; - - private String fqdn; - private Constructor constructorCache; - - public GenericClassSchema(String name, List fields, String namespace, List imports) - { - super(name, namespace, imports); - this.fields = fields; - if(namespace == null) { - this.fqdn = name; - } else { - this.fqdn = namespace+"."+name; - } - } - - //@Override - //public String getFullName() - //{ - // if(namespace == null) { - // return getName(); - // } else { - // return namespace+"."+getName(); - // } - //} - - public List getFields() - { - return fields; - } - - public String getNamespace() - { - return namespace; - } - - @Override - public String getExpression() - { - StringBuffer b = new StringBuffer(); - b.append("(class "); - b.append(getName()); - if(namespace != null) { - b.append(" (package "+namespace+")"); - } - for(GenericFieldSchema f : fields) { - b.append(" "+f.getExpression()); - } - b.append(")"); - return b.toString(); - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - // FIXME - } - - @Override - @SuppressWarnings("unchecked") - public Object convert(GenericObject obj) - { - // FIXME - return obj; - } - - @Override - public Object newInstance() - { - return new HashMap(fields.size()); - } -} - diff --git a/java-plan2/src/org/msgpack/schema/GenericFieldSchema.java b/java-plan2/src/org/msgpack/schema/GenericFieldSchema.java deleted file mode 100644 index 507ee18..0000000 --- a/java-plan2/src/org/msgpack/schema/GenericFieldSchema.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.msgpack.schema; - -import java.util.Map; -import java.lang.reflect.Field; - -public final class GenericFieldSchema extends FieldSchema { - public GenericFieldSchema(String name, Schema type) - { - super(name, type); - } - - @Override - public Object getFieldValue(Object obj) - { - return ((Map)obj).get(getName()); - } - - @Override - @SuppressWarnings("unchecked") - public void setFieldValue(Object obj, Object value) - { - ((Map)obj).put(getName(), value); - } -} - diff --git a/java-plan2/src/org/msgpack/schema/IntSchema.java b/java-plan2/src/org/msgpack/schema/IntSchema.java deleted file mode 100644 index 69771c3..0000000 --- a/java-plan2/src/org/msgpack/schema/IntSchema.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.*; - -public class IntSchema extends Schema { - public IntSchema() - { - super("Integer"); - } - - @Override - public String getExpression() - { - return "int"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - pk.packInt((Integer)obj); - } - - @Override - public Object convert(GenericObject obj) - { - return obj.asInt(); - } - - @Override - public Object createByte(byte v) - { - return (int)v; - } - - @Override - public Object createShort(short v) - { - return (int)v; - } - - @Override - public Object createInt(int v) - { - return (int)v; - } - - @Override - public Object createLong(long v) - { - return (int)v; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/LongSchema.java b/java-plan2/src/org/msgpack/schema/LongSchema.java deleted file mode 100644 index 0ba3057..0000000 --- a/java-plan2/src/org/msgpack/schema/LongSchema.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.*; - -public class LongSchema extends Schema { - public LongSchema() - { - super("Long"); - } - - @Override - public String getExpression() - { - return "long"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - pk.packLong((Long)obj); - } - - @Override - public Object convert(GenericObject obj) - { - return obj.asLong(); - } - - @Override - public Object createByte(byte v) - { - return (long)v; - } - - @Override - public Object createShort(short v) - { - return (long)v; - } - - @Override - public Object createInt(int v) - { - return (long)v; - } - - @Override - public Object createLong(long v) - { - return (long)v; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/MapSchema.java b/java-plan2/src/org/msgpack/schema/MapSchema.java deleted file mode 100644 index e72cf63..0000000 --- a/java-plan2/src/org/msgpack/schema/MapSchema.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.msgpack.schema; - -import java.util.List; -import java.util.Map; -import java.util.HashMap; -import java.io.IOException; -import org.msgpack.*; - -public class MapSchema extends Schema { - private Schema keyType; - private Schema valueType; - - public MapSchema(Schema keyType, Schema valueType) - { - super("map"); - this.keyType = keyType; - this.valueType = valueType; - } - - public Schema getKeyType() - { - return keyType; - } - - public Schema getValueType() - { - return valueType; - } - - @Override - public String getFullName() - { - return "HashList<"+keyType.getFullName()+", "+valueType.getFullName()+">"; - } - - @Override - public String getExpression() - { - return "(map "+keyType.getExpression()+" "+valueType.getExpression()+")"; - } - - @Override - @SuppressWarnings("unchecked") - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - Map d = (Map)obj; - pk.packMap(d.size()); - for(Map.Entry e : d.entrySet()) { - keyType.pack(pk, e.getKey()); - valueType.pack(pk, e.getValue()); - } - } - - @Override - @SuppressWarnings("unchecked") - public Object convert(GenericObject obj) - { - Map d = obj.asMap(); - Map g = new HashMap(); - for(Map.Entry e : d.entrySet()) { - g.put(keyType.convert(e.getKey()), valueType.convert(e.getValue())); - } - return g; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/ObjectSchema.java b/java-plan2/src/org/msgpack/schema/ObjectSchema.java deleted file mode 100644 index a9f30f5..0000000 --- a/java-plan2/src/org/msgpack/schema/ObjectSchema.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.*; - -public class ObjectSchema extends Schema { - public ObjectSchema() - { - super("object"); - } - - public String getFullName() - { - return "GenericObject"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - pk.pack(obj); - } - - @Override - public Object convert(GenericObject obj) - { - return obj; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/PrimitiveSchema.java b/java-plan2/src/org/msgpack/schema/PrimitiveSchema.java deleted file mode 100644 index 023d81b..0000000 --- a/java-plan2/src/org/msgpack/schema/PrimitiveSchema.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.msgpack.schema; - -public abstract class PrimitiveSchema extends Schema { - public static enum PrimitiveType { - BYTE, - SHORT, - INT, - LONG, - FLOAT, - DOUBLE, - } - - public final PrimitiveType type; - - protected PrimitiveSchema(String name, PrimitiveType type) - { - super(name); - this.type = type; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/RawSchema.java b/java-plan2/src/org/msgpack/schema/RawSchema.java deleted file mode 100644 index 847ad29..0000000 --- a/java-plan2/src/org/msgpack/schema/RawSchema.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.msgpack.schema; - -import java.io.IOException; -import java.nio.charset.Charset; -import org.msgpack.*; - -public class RawSchema extends Schema { - public RawSchema() - { - super("raw"); - } - - public String getFullName() - { - return "byte[]"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - byte[] d = (byte[])obj; - pk.packRaw(d.length); - pk.packRawBody(d); - } - - @Override - public Object convert(GenericObject obj) - { - return obj.asBytes(); - } - - @Override - public Object createRaw(byte[] b, int offset, int length) - { - byte[] d = new byte[length]; - System.arraycopy(b, offset, d, 0, length); - return d; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/SSchemaParser.java b/java-plan2/src/org/msgpack/schema/SSchemaParser.java deleted file mode 100644 index bfe912f..0000000 --- a/java-plan2/src/org/msgpack/schema/SSchemaParser.java +++ /dev/null @@ -1,246 +0,0 @@ -package org.msgpack.schema; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Stack; -import java.util.regex.Pattern; -import java.util.regex.Matcher; - -// FIXME exception class - -class SSchemaParser { - public static Schema parse(String source) - { - return new SSchemaParser(false).run(source); - } - - public static Schema load(String source) - { - return new SSchemaParser(true).run(source); - } - - private static abstract class SExp { - boolean isAtom() { return false; } - public String getAtom() { return null; } - - boolean isTuple() { return false; } - public SExp getTuple(int i) { return null; } - public int size() { return 0; } - public boolean empty() { return size() == 0; } - Iterator iterator(int offset) { return null; } - } - - private static class SAtom extends SExp { - private String atom; - - SAtom(String atom) { this.atom = atom; } - - boolean isAtom() { return true; } - public String getAtom() { return atom; } - - public String toString() { return atom; } - } - - private static class STuple extends SExp { - private List tuple; - - STuple() { this.tuple = new ArrayList(); } - - public void add(SExp e) { tuple.add(e); } - - boolean isTuple() { return true; } - public SExp getTuple(int i) { return tuple.get(i); } - public int size() { return tuple.size(); } - - Iterator iterator(int skip) { - Iterator i = tuple.iterator(); - for(int s=0; s < skip; ++s) { i.next(); } - return i; - } - - public String toString() { - if(tuple.isEmpty()) { return "()"; } - Iterator i = tuple.iterator(); - StringBuffer o = new StringBuffer(); - o.append("(").append(i.next()); - while(i.hasNext()) { o.append(" ").append(i.next()); } - o.append(")"); - return o.toString(); - } - } - - boolean specificClass; - - private SSchemaParser(boolean specificClass) - { - this.specificClass = specificClass; - } - - private static Pattern pattern = Pattern.compile( - "(?:\\s+)|([\\(\\)]|[\\d\\w\\.]+)"); - - private Schema run(String source) - { - Matcher m = pattern.matcher(source); - - Stack stack = new Stack(); - String token; - - while(true) { - while(true) { - if(!m.find()) { throw new RuntimeException("unexpected end of file"); } - token = m.group(1); - if(token != null) { break; } - } - - if(token.equals("(")) { - stack.push(new STuple()); - } else if(token.equals(")")) { - STuple top = stack.pop(); - if(stack.empty()) { - stack.push(top); - break; - } - stack.peek().add(top); - } else { - if(stack.empty()) { - throw new RuntimeException("unexpected token '"+token+"'"); - } - stack.peek().add(new SAtom(token)); - } - } - - while(true) { - if(!m.find()) { break; } - token = m.group(1); - if(token != null) { throw new RuntimeException("unexpected token '"+token+"'"); } - } - - return readType( stack.pop() ); - } - - private Schema readType(SExp exp) - { - if(exp.isAtom()) { - String type = exp.getAtom(); - // FIXME - if(type.equals("string")) { - return new StringSchema(); - } else if(type.equals("raw")) { - return new RawSchema(); - } else if(type.equals("short")) { - return new ShortSchema(); - } else if(type.equals("int")) { - return new IntSchema(); - } else if(type.equals("long")) { - return new LongSchema(); - } else if(type.equals("object")) { - return new ObjectSchema(); - } else { - throw new RuntimeException("byte, short, int, long, float, double, raw, string or object is expected but got '"+type+"': "+exp); - } - } else { - String type = exp.getTuple(0).getAtom(); - if(type.equals("class")) { - return parseClass(exp); - } else if(type.equals("array")) { - return parseArray(exp); - } else if(type.equals("map")) { - return parseMap(exp); - } else { - throw new RuntimeException("class, array or map is expected but got '"+type+"': "+exp); - } - } - } - - private ClassSchema parseClass(SExp exp) - { - if(exp.size() < 3 || !exp.getTuple(1).isAtom()) { - throw new RuntimeException("class is (class NAME CLASS_BODY): "+exp); - } - - String namespace = null; - List imports = new ArrayList(); - String name = exp.getTuple(1).getAtom(); - List fields = new ArrayList(); - - for(Iterator i=exp.iterator(2); i.hasNext();) { - SExp subexp = i.next(); - if(!subexp.isTuple() || subexp.empty() || !subexp.getTuple(0).isAtom()) { - throw new RuntimeException("field, package or import is expected: "+subexp); - } - String type = subexp.getTuple(0).getAtom(); - if(type.equals("field")) { - fields.add( parseField(subexp) ); - } else if(type.equals("package")) { - if(namespace != null) { - throw new RuntimeException("duplicated package definition: "+subexp); - } - namespace = parseNamespace(subexp); - } else if(type.equals("import")) { - imports.add( parseImport(subexp) ); - } else { - throw new RuntimeException("field, package or import is expected but got '"+type+"': "+subexp); - } - } - - if(specificClass) { - return new SpecificClassSchema(name, fields, namespace, imports); - } else { - return new GenericClassSchema(name, fields, namespace, imports); - } - } - - private ArraySchema parseArray(SExp exp) - { - if(exp.size() != 2) { - throw new RuntimeException("array is (array ELEMENT_TYPE): "+exp); - } - Schema elementType = readType(exp.getTuple(1)); - return new ArraySchema(elementType); - } - - private MapSchema parseMap(SExp exp) - { - if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { - throw new RuntimeException("map is (map KEY_TYPE VALUE_TYPE): "+exp); - } - Schema keyType = readType(exp.getTuple(1)); - Schema valueType = readType(exp.getTuple(2)); - return new MapSchema(keyType, valueType); - } - - private String parseNamespace(SExp exp) - { - if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { - throw new RuntimeException("package is (package NAME): "+exp); - } - String name = exp.getTuple(1).getAtom(); - return name; - } - - private String parseImport(SExp exp) - { - if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { - throw new RuntimeException("import is (import NAME): "+exp); - } - String name = exp.getTuple(1).getAtom(); - return name; - } - - private FieldSchema parseField(SExp exp) - { - if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { - throw new RuntimeException("field is (field NAME TYPE): "+exp); - } - String name = exp.getTuple(1).getAtom(); - Schema type = readType(exp.getTuple(2)); - if(specificClass) { - return new SpecificFieldSchema(name, type); - } else { - return new GenericFieldSchema(name, type); - } - } -} - diff --git a/java-plan2/src/org/msgpack/schema/Schema.java b/java-plan2/src/org/msgpack/schema/Schema.java deleted file mode 100644 index 15b7e72..0000000 --- a/java-plan2/src/org/msgpack/schema/Schema.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.impl.*; -import org.msgpack.Packer; -import org.msgpack.GenericObject; - -public abstract class Schema { - private String expression; - private String name; - - public Schema(String name) - { - this.expression = expression; - this.name = name; - } - - public String getName() - { - return name; - } - - public String getFullName() - { - return name; - } - - public String getExpression() - { - return name; - } - - public static Schema parse(String source) - { - return SSchemaParser.parse(source); - } - - public static Schema load(String source) - { - return SSchemaParser.load(source); - } - - public abstract void pack(Packer pk, Object obj) throws IOException; - public abstract Object convert(GenericObject obj); - //public abstract Object convertGeneric(GenericObject obj); - - - public Object createNil() - { - return null; - } - - public Object createBoolean(boolean v) - { - throw new RuntimeException("type error"); - } - - public Object createByte(byte v) - { - throw new RuntimeException("type error"); - } - - public Object createShort(short v) - { - throw new RuntimeException("type error"); - } - - public Object createInt(int v) - { - throw new RuntimeException("type error"); - } - - public Object createLong(long v) - { - throw new RuntimeException("type error"); - } - - public Object createFloat(float v) - { - throw new RuntimeException("type error"); - } - - public Object createDouble(double v) - { - throw new RuntimeException("type error"); - } - - public Object createRaw(byte[] b, int offset, int length) - { - throw new RuntimeException("type error"); - } -} - diff --git a/java-plan2/src/org/msgpack/schema/ShortSchema.java b/java-plan2/src/org/msgpack/schema/ShortSchema.java deleted file mode 100644 index aa95f51..0000000 --- a/java-plan2/src/org/msgpack/schema/ShortSchema.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.*; - -public class ShortSchema extends Schema { - public ShortSchema() - { - super("Short"); - } - - @Override - public String getExpression() - { - return "short"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - pk.packShort((Short)obj); - } - - @Override - public Object convert(GenericObject obj) - { - return obj.asShort(); - } - - @Override - public Object createByte(byte v) - { - return (int)v; - } - - @Override - public Object createShort(short v) - { - return (int)v; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/SpecificClassSchema.java b/java-plan2/src/org/msgpack/schema/SpecificClassSchema.java deleted file mode 100644 index 75c474a..0000000 --- a/java-plan2/src/org/msgpack/schema/SpecificClassSchema.java +++ /dev/null @@ -1,156 +0,0 @@ -package org.msgpack.schema; - -import java.util.List; -import java.util.Map; -import java.io.IOException; -import java.util.Iterator; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import org.msgpack.*; - -public class SpecificClassSchema extends ClassSchema { - private List fields; - private String namespace; - private List imports; - - private String fqdn; - private Constructor constructorCache; - - public SpecificClassSchema(String name, List fields, String namespace, List imports) - { - super(name, namespace, imports); - this.fields = fields; - if(namespace == null) { - this.fqdn = name; - } else { - this.fqdn = namespace+"."+name; - } - } - - //@Override - //public String getFullName() - //{ - // if(namespace == null) { - // return getName(); - // } else { - // return namespace+"."+getName(); - // } - //} - - public List getFields() - { - return fields; - } - - @Override - public String getExpression() - { - StringBuffer b = new StringBuffer(); - b.append("(class "); - b.append(getName()); - if(namespace != null) { - b.append(" (package "+namespace+")"); - } - for(SpecificFieldSchema f : fields) { - b.append(" "+f.getExpression()); - } - b.append(")"); - return b.toString(); - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - - if(constructorCache == null) { - cacheConstructor(); - } - - pk.packArray(fields.size()); - for(SpecificFieldSchema f : fields) { - f.getType().pack(pk, f.getFieldValue(obj)); - } - } - - @Override - @SuppressWarnings("unchecked") - public Object convert(GenericObject obj) - { - if(constructorCache == null) { - cacheConstructor(); - } - - List d = obj.asArray(); - - try { - Object g = constructorCache.newInstance((Object[])null); - - Iterator vi = d.iterator(); - Iterator fi = fields.iterator(); - while(fi.hasNext() && vi.hasNext()) { - SpecificFieldSchema f = fi.next(); - GenericObject v = vi.next(); - f.setFieldValue(g, f.getType().convert(v)); - } - // leave it as uninitialized - //while(fi.hasNext()) { - // SpecificFieldSchema f = fi.next(); - // g.put(f.getName(), null); - //} - - return g; - - } catch (InvocationTargetException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } catch (InstantiationException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } catch (IllegalAccessException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } - } - - private void cacheConstructor() - { - try { - Class c = Class.forName(fqdn); - int index = 0; - for(SpecificFieldSchema f : fields) { - f.cacheField(c, index++); - } - constructorCache = c.getDeclaredConstructor((Class[])null); - constructorCache.setAccessible(true); - } catch(ClassNotFoundException e) { - throw new RuntimeException("class not found: "+fqdn); - } catch (NoSuchMethodException e) { - throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); - } - } - - @Override - public Object newInstance() - { - if(constructorCache == null) { - cacheConstructor(); - } - try { - return constructorCache.newInstance((Object[])null); - } catch (InvocationTargetException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } catch (InstantiationException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } catch (IllegalAccessException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } - } - - public boolean equals(SpecificClassSchema o) - { - return (namespace != null ? namespace.equals(o.getNamespace()) : o.getNamespace() == null) && - getName().equals(o.getName()); - } -} - diff --git a/java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java b/java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java deleted file mode 100644 index 297df27..0000000 --- a/java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java +++ /dev/null @@ -1,78 +0,0 @@ -package org.msgpack.schema; - -import java.util.Map; -import java.util.Arrays; -import java.lang.reflect.Field; -import org.msgpack.*; - -public class SpecificFieldSchema extends FieldSchema { - public Field fieldCache; - private int index; - - public SpecificFieldSchema(String name, Schema type) - { - super(name, type); - this.index = -1; - } - - @Override - public Object getFieldValue(Object obj) - { - if(index >= 0) { - return ((MessageMergeable)obj).getField(index); - } - - try { - return fieldCache.get(obj); - } catch(IllegalArgumentException e) { - throw new RuntimeException("can't get value from '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); - } catch(IllegalAccessException e) { - throw new RuntimeException("can't get value from '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); - } - } - - @Override - public void setFieldValue(Object obj, Object value) - { - if(index >= 0) { - ((MessageMergeable)obj).setField(index, value); - return; - } - - try { - fieldCache.set(obj, value); - } catch(IllegalArgumentException e) { - throw new RuntimeException("can't set value into '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); - } catch(IllegalAccessException e) { - throw new RuntimeException("can't set value into '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); - } - } - - void cacheField(Class c, int index) - { - for(Class i : c.getInterfaces()) { - if(i.equals(MessageMergeable.class)) { - this.index = index; - return; - } - } - - try { - fieldCache = c.getDeclaredField(getName()); - if(!fieldCache.isAccessible()) { - fieldCache.setAccessible(true); - } - } catch(NoSuchFieldException e) { - throw new RuntimeException("can't get '"+getName()+"' field of '"+c.getName()+"' class: "+e.getMessage()); - } catch(SecurityException e) { - throw new RuntimeException("can't get '"+getName()+"' field of '"+c.getName()+"' class: "+e.getMessage()); - } - } - - //public void setFieldInt(Object obj, int value) - //{ - // if(type instanceof PrimitiveSchema) { - // } - //} -} - diff --git a/java-plan2/src/org/msgpack/schema/StringSchema.java b/java-plan2/src/org/msgpack/schema/StringSchema.java deleted file mode 100644 index fc6855b..0000000 --- a/java-plan2/src/org/msgpack/schema/StringSchema.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.msgpack.schema; - -import java.io.IOException; -import java.nio.charset.Charset; -import org.msgpack.*; - -public class StringSchema extends Schema { - public StringSchema() - { - super("string"); - } - - public String getFullName() - { - return "String"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - String s = (String)obj; - byte[] d = s.getBytes("UTF-8"); - pk.packRaw(d.length); - pk.packRawBody(d); - } - - @Override - public Object convert(GenericObject obj) - { - return obj.asString(); - } - - @Override - public Object createRaw(byte[] b, int offset, int length) - { - try { - return new String(b, offset, length, "UTF-8"); // XXX FIXME debug - } catch (Exception e) { - // FIXME - throw new RuntimeException(e.getMessage()); - } - } -} - diff --git a/java-plan2/test/Generate.java b/java-plan2/test/Generate.java deleted file mode 100644 index 6b7800b..0000000 --- a/java-plan2/test/Generate.java +++ /dev/null @@ -1,18 +0,0 @@ -import java.io.*; -import java.util.*; -import org.msgpack.*; -import org.msgpack.schema.*; - -public class Generate { - public static void main(String[] args) throws IOException - { - Writer output = new OutputStreamWriter(System.out); - - Schema s1 = Schema.parse("(class Test (field uri raw) (field width int))"); - ClassGenerator.write(s1, output); - - Schema s2 = Schema.parse("(class MediaContent (package serializers.msgpack) (field image (array (class Image (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); - ClassGenerator.write(s2, output); - } -} - diff --git a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java deleted file mode 100644 index ecb64fd..0000000 --- a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java +++ /dev/null @@ -1,277 +0,0 @@ -package serializers.msgpack; - -import java.util.*; -import java.io.*; -import org.msgpack.*; -import org.msgpack.schema.*; - -public final class MediaContent implements MessagePackable, MessageConvertable, MessageMergeable -{ - private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class MediaContent (package serializers.msgpack) (field image (array (class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); - public static ClassSchema getSchema() { return _SCHEMA; } - - public ArrayList image; - public Media media; - - public MediaContent() { } - - @Override - public void messagePack(Packer pk) throws IOException - { - List _f = _SCHEMA.getFields(); - pk.packArray(2); - _f.get(0).getType().pack(pk, image); - _f.get(1).getType().pack(pk, media); - } - - @Override - @SuppressWarnings("unchecked") - public void messageConvert(GenericObject obj) - { - List _l = obj.asArray(); - List _f = _SCHEMA.getFields(); - if(_l.size() <= 0) { return; } image = (ArrayList)_f.get(0).getType().convert(_l.get(0)); - if(_l.size() <= 1) { return; } media = (Media)_f.get(1).getType().convert(_l.get(1)); - } - - public static MediaContent convert(GenericObject obj) - { - return (MediaContent)_SCHEMA.convert(obj); - } - - public void setField(int index, Object value) - { - switch(index) { - case 0: - image = (ArrayList)value; - break; - case 1: - media = (Media)value; - break; - } - } - - public Object getField(int index) - { - switch(index) { - case 0: - return image; - case 1: - return media; - } - return null; - } -} - -final class Image implements MessagePackable, MessageConvertable, MessageMergeable -{ - private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int))"); - public static ClassSchema getSchema() { return _SCHEMA; } - - public String uri; - public String title; - public Integer width; - public Integer height; - public Integer size; - - public Image() { } - - @Override - public void messagePack(Packer pk) throws IOException - { - List _f = _SCHEMA.getFields(); - pk.packArray(5); - _f.get(0).getType().pack(pk, uri); - _f.get(1).getType().pack(pk, title); - _f.get(2).getType().pack(pk, width); - _f.get(3).getType().pack(pk, height); - _f.get(4).getType().pack(pk, size); - } - - @Override - @SuppressWarnings("unchecked") - public void messageConvert(GenericObject obj) - { - List _l = obj.asArray(); - List _f = _SCHEMA.getFields(); - if(_l.size() <= 0) { return; } uri = (String)_f.get(0).getType().convert(_l.get(0)); - if(_l.size() <= 1) { return; } title = (String)_f.get(1).getType().convert(_l.get(1)); - if(_l.size() <= 2) { return; } width = (Integer)_f.get(2).getType().convert(_l.get(2)); - if(_l.size() <= 3) { return; } height = (Integer)_f.get(3).getType().convert(_l.get(3)); - if(_l.size() <= 4) { return; } size = (Integer)_f.get(4).getType().convert(_l.get(4)); - } - - public static Image convert(GenericObject obj) - { - return (Image)_SCHEMA.convert(obj); - } - - public void setField(int index, Object value) - { - switch(index) { - case 0: - uri = (String)value; - break; - case 1: - title = (String)value; - break; - case 2: - width = (Integer)value; - break; - case 3: - height = (Integer)value; - break; - case 4: - size = (Integer)value; - break; - } - } - - public Object getField(int index) - { - switch(index) { - case 0: - return uri; - case 1: - return title; - case 2: - return width; - case 3: - return height; - case 4: - return size; - } - return null; - } -} - -final class Media implements MessagePackable, MessageConvertable, MessageMergeable -{ - private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))"); - public static ClassSchema getSchema() { return _SCHEMA; } - - public String uri; - public String title; - public Integer width; - public Integer height; - public String format; - public Long duration; - public Long size; - public Integer bitrate; - public ArrayList person; - public Integer player; - public String copyright; - - public Media() { } - - @Override - public void messagePack(Packer pk) throws IOException - { - List _f = _SCHEMA.getFields(); - pk.packArray(11); - _f.get(0).getType().pack(pk, uri); - _f.get(1).getType().pack(pk, title); - _f.get(2).getType().pack(pk, width); - _f.get(3).getType().pack(pk, height); - _f.get(4).getType().pack(pk, format); - _f.get(5).getType().pack(pk, duration); - _f.get(6).getType().pack(pk, size); - _f.get(7).getType().pack(pk, bitrate); - _f.get(8).getType().pack(pk, person); - _f.get(9).getType().pack(pk, player); - _f.get(10).getType().pack(pk, copyright); - } - - @Override - @SuppressWarnings("unchecked") - public void messageConvert(GenericObject obj) - { - List _l = obj.asArray(); - List _f = _SCHEMA.getFields(); - if(_l.size() <= 0) { return; } uri = (String)_f.get(0).getType().convert(_l.get(0)); - if(_l.size() <= 1) { return; } title = (String)_f.get(1).getType().convert(_l.get(1)); - if(_l.size() <= 2) { return; } width = (Integer)_f.get(2).getType().convert(_l.get(2)); - if(_l.size() <= 3) { return; } height = (Integer)_f.get(3).getType().convert(_l.get(3)); - if(_l.size() <= 4) { return; } format = (String)_f.get(4).getType().convert(_l.get(4)); - if(_l.size() <= 5) { return; } duration = (Long)_f.get(5).getType().convert(_l.get(5)); - if(_l.size() <= 6) { return; } size = (Long)_f.get(6).getType().convert(_l.get(6)); - if(_l.size() <= 7) { return; } bitrate = (Integer)_f.get(7).getType().convert(_l.get(7)); - if(_l.size() <= 8) { return; } person = (ArrayList)_f.get(8).getType().convert(_l.get(8)); - if(_l.size() <= 9) { return; } player = (Integer)_f.get(9).getType().convert(_l.get(9)); - if(_l.size() <= 10) { return; } copyright = (String)_f.get(10).getType().convert(_l.get(10)); - } - - public static Media convert(GenericObject obj) - { - return (Media)_SCHEMA.convert(obj); - } - - public void setField(int index, Object value) - { - switch(index) { - case 0: - uri = (String)value; - break; - case 1: - title = (String)value; - break; - case 2: - width = (Integer)value; - break; - case 3: - height = (Integer)value; - break; - case 4: - format = (String)value; - break; - case 5: - duration = (Long)value; - break; - case 6: - size = (Long)value; - break; - case 7: - bitrate = (Integer)value; - break; - case 8: - person = (ArrayList)value; - break; - case 9: - player = (Integer)value; - break; - case 10: - copyright = (String)value; - break; - } - } - - public Object getField(int index) - { - switch(index) { - case 0: - return uri; - case 1: - return title; - case 2: - return width; - case 3: - return height; - case 4: - return format; - case 5: - return duration; - case 6: - return size; - case 7: - return bitrate; - case 8: - return person; - case 9: - return player; - case 10: - return copyright; - } - return null; - } - -} diff --git a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java deleted file mode 100644 index acb5580..0000000 --- a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java +++ /dev/null @@ -1,70 +0,0 @@ -package serializers.msgpack; - -import java.io.*; -import java.util.*; -import java.nio.charset.Charset; - -import org.msgpack.*; -import org.msgpack.schema.*; -import serializers.msgpack.*; - -import serializers.ObjectSerializer; - -public class MessagePackSerializer implements ObjectSerializer -{ - public String getName() { - return "msgpack-specific"; - } - - public MediaContent create() throws Exception { - Media media = new Media(); - media.uri = "http://javaone.com/keynote.mpg"; - media.format = "video/mpg4"; - media.title = "Javaone Keynote"; - media.duration = 1234567L; - media.bitrate = 0; - media.person = new ArrayList(2); - media.person.add("Bill Gates"); - media.person.add("Steve Jobs"); - media.player = 0; - media.height = 0; - media.width = 0; - media.size = 123L; - media.copyright = ""; - - Image image1 = new Image(); - image1.uri = "http://javaone.com/keynote_large.jpg"; - image1.width = 0; - image1.height = 0; - image1.size = 2; - image1.title = "Javaone Keynote"; - - Image image2 = new Image(); - image2.uri = "http://javaone.com/keynote_thumbnail.jpg"; - image2.width = 0; - image2.height = 0; - image2.size = 1; - image2.title = "Javaone Keynote"; - - MediaContent content = new MediaContent(); - content.media = media; - content.image = new ArrayList(2); - content.image.add(image1); - content.image.add(image2); - return content; - } - - public MediaContent deserialize(byte[] array) throws Exception { - UnbufferedUnpacker pac = new UnbufferedUnpacker().useSchema(MediaContent.getSchema()); - pac.execute(array); - return (MediaContent)pac.getData(); - } - - public byte[] serialize(MediaContent content) throws Exception { - ByteArrayOutputStream os = new ByteArrayOutputStream(); - Packer pk = new Packer(os); - pk.pack(content); - return os.toByteArray(); - } -} - diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs deleted file mode 100644 index 547ba48..0000000 --- a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs +++ /dev/null @@ -1,21 +0,0 @@ -(class MediaContent - (package serializers.msgpack) - (field image (array (class Image - (field uri string) - (field title string) - (field width int) - (field height int) - (field size int)))) - (field media (class Media - (field uri string) - (field title string) - (field width int) - (field height int) - (field format string) - (field duration long) - (field size long) - (field bitrate int) - (field person (array string)) - (field player int) - (field copyright string))) - ) diff --git a/java-plan3/build.xml b/java/build.xml similarity index 78% rename from java-plan3/build.xml rename to java/build.xml index 598a853..40d7d61 100644 --- a/java-plan3/build.xml +++ b/java/build.xml @@ -12,4 +12,7 @@ + + + diff --git a/java-plan3/src/org/msgpack/MessageMergeable.java b/java/src/org/msgpack/MessageMergeable.java similarity index 94% rename from java-plan3/src/org/msgpack/MessageMergeable.java rename to java/src/org/msgpack/MessageMergeable.java index e11119c..e5a5b45 100644 --- a/java-plan3/src/org/msgpack/MessageMergeable.java +++ b/java/src/org/msgpack/MessageMergeable.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/MessagePackable.java b/java/src/org/msgpack/MessagePackable.java similarity index 94% rename from java-plan3/src/org/msgpack/MessagePackable.java rename to java/src/org/msgpack/MessagePackable.java index d8a7db9..9e9852a 100644 --- a/java-plan3/src/org/msgpack/MessagePackable.java +++ b/java/src/org/msgpack/MessagePackable.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/MessageTypeException.java b/java/src/org/msgpack/MessageTypeException.java similarity index 96% rename from java-plan3/src/org/msgpack/MessageTypeException.java rename to java/src/org/msgpack/MessageTypeException.java index 09031b2..feb6c08 100644 --- a/java-plan3/src/org/msgpack/MessageTypeException.java +++ b/java/src/org/msgpack/MessageTypeException.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/Packer.java b/java/src/org/msgpack/Packer.java similarity index 99% rename from java-plan3/src/org/msgpack/Packer.java rename to java/src/org/msgpack/Packer.java index 7f2508c..ebd8402 100644 --- a/java-plan3/src/org/msgpack/Packer.java +++ b/java/src/org/msgpack/Packer.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/Schema.java b/java/src/org/msgpack/Schema.java similarity index 98% rename from java-plan3/src/org/msgpack/Schema.java rename to java/src/org/msgpack/Schema.java index f99b3d0..f191f7a 100644 --- a/java-plan3/src/org/msgpack/Schema.java +++ b/java/src/org/msgpack/Schema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/UnbufferedUnpacker.java b/java/src/org/msgpack/UnbufferedUnpacker.java similarity index 97% rename from java-plan3/src/org/msgpack/UnbufferedUnpacker.java rename to java/src/org/msgpack/UnbufferedUnpacker.java index 471605f..b427973 100644 --- a/java-plan3/src/org/msgpack/UnbufferedUnpacker.java +++ b/java/src/org/msgpack/UnbufferedUnpacker.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/UnpackException.java b/java/src/org/msgpack/UnpackException.java similarity index 94% rename from java-plan3/src/org/msgpack/UnpackException.java rename to java/src/org/msgpack/UnpackException.java index db08d95..35e3e44 100644 --- a/java-plan3/src/org/msgpack/UnpackException.java +++ b/java/src/org/msgpack/UnpackException.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/UnpackIterator.java b/java/src/org/msgpack/UnpackIterator.java similarity index 96% rename from java-plan3/src/org/msgpack/UnpackIterator.java rename to java/src/org/msgpack/UnpackIterator.java index 9975b68..0a78e83 100644 --- a/java-plan3/src/org/msgpack/UnpackIterator.java +++ b/java/src/org/msgpack/UnpackIterator.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/Unpacker.java b/java/src/org/msgpack/Unpacker.java similarity index 99% rename from java-plan3/src/org/msgpack/Unpacker.java rename to java/src/org/msgpack/Unpacker.java index af211c6..458ffee 100644 --- a/java-plan3/src/org/msgpack/Unpacker.java +++ b/java/src/org/msgpack/Unpacker.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/impl/UnpackerImpl.java b/java/src/org/msgpack/impl/UnpackerImpl.java similarity index 95% rename from java-plan3/src/org/msgpack/impl/UnpackerImpl.java rename to java/src/org/msgpack/impl/UnpackerImpl.java index 47a1800..ddf9940 100644 --- a/java-plan3/src/org/msgpack/impl/UnpackerImpl.java +++ b/java/src/org/msgpack/impl/UnpackerImpl.java @@ -1,3 +1,20 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.impl; import java.nio.ByteBuffer; diff --git a/java-plan3/src/org/msgpack/schema/ArraySchema.java b/java/src/org/msgpack/schema/ArraySchema.java similarity index 98% rename from java-plan3/src/org/msgpack/schema/ArraySchema.java rename to java/src/org/msgpack/schema/ArraySchema.java index 24fa758..fd47143 100644 --- a/java-plan3/src/org/msgpack/schema/ArraySchema.java +++ b/java/src/org/msgpack/schema/ArraySchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/ByteSchema.java b/java/src/org/msgpack/schema/ByteSchema.java similarity index 97% rename from java-plan3/src/org/msgpack/schema/ByteSchema.java rename to java/src/org/msgpack/schema/ByteSchema.java index 3bc7045..9ee6a82 100644 --- a/java-plan3/src/org/msgpack/schema/ByteSchema.java +++ b/java/src/org/msgpack/schema/ByteSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/ClassGenerator.java b/java/src/org/msgpack/schema/ClassGenerator.java similarity index 99% rename from java-plan3/src/org/msgpack/schema/ClassGenerator.java rename to java/src/org/msgpack/schema/ClassGenerator.java index 65213aa..061dcbb 100644 --- a/java-plan3/src/org/msgpack/schema/ClassGenerator.java +++ b/java/src/org/msgpack/schema/ClassGenerator.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/ClassSchema.java b/java/src/org/msgpack/schema/ClassSchema.java similarity index 98% rename from java-plan3/src/org/msgpack/schema/ClassSchema.java rename to java/src/org/msgpack/schema/ClassSchema.java index 75315e7..cd5c008 100644 --- a/java-plan3/src/org/msgpack/schema/ClassSchema.java +++ b/java/src/org/msgpack/schema/ClassSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/DoubleSchema.java b/java/src/org/msgpack/schema/DoubleSchema.java similarity index 97% rename from java-plan3/src/org/msgpack/schema/DoubleSchema.java rename to java/src/org/msgpack/schema/DoubleSchema.java index feffbb9..d53e47d 100644 --- a/java-plan3/src/org/msgpack/schema/DoubleSchema.java +++ b/java/src/org/msgpack/schema/DoubleSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/FieldSchema.java b/java/src/org/msgpack/schema/FieldSchema.java similarity index 95% rename from java-plan3/src/org/msgpack/schema/FieldSchema.java rename to java/src/org/msgpack/schema/FieldSchema.java index 3391f2b..66c2ff2 100644 --- a/java-plan3/src/org/msgpack/schema/FieldSchema.java +++ b/java/src/org/msgpack/schema/FieldSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/FloatSchema.java b/java/src/org/msgpack/schema/FloatSchema.java similarity index 97% rename from java-plan3/src/org/msgpack/schema/FloatSchema.java rename to java/src/org/msgpack/schema/FloatSchema.java index 2f4240a..2777521 100644 --- a/java-plan3/src/org/msgpack/schema/FloatSchema.java +++ b/java/src/org/msgpack/schema/FloatSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/GenericClassSchema.java b/java/src/org/msgpack/schema/GenericClassSchema.java similarity index 98% rename from java-plan3/src/org/msgpack/schema/GenericClassSchema.java rename to java/src/org/msgpack/schema/GenericClassSchema.java index 736bdfa..ffdd4ab 100644 --- a/java-plan3/src/org/msgpack/schema/GenericClassSchema.java +++ b/java/src/org/msgpack/schema/GenericClassSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/GenericSchema.java b/java/src/org/msgpack/schema/GenericSchema.java similarity index 98% rename from java-plan3/src/org/msgpack/schema/GenericSchema.java rename to java/src/org/msgpack/schema/GenericSchema.java index 52e0161..0adf898 100644 --- a/java-plan3/src/org/msgpack/schema/GenericSchema.java +++ b/java/src/org/msgpack/schema/GenericSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/IArraySchema.java b/java/src/org/msgpack/schema/IArraySchema.java similarity index 94% rename from java-plan3/src/org/msgpack/schema/IArraySchema.java rename to java/src/org/msgpack/schema/IArraySchema.java index ccc6d14..67e9f55 100644 --- a/java-plan3/src/org/msgpack/schema/IArraySchema.java +++ b/java/src/org/msgpack/schema/IArraySchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/IMapSchema.java b/java/src/org/msgpack/schema/IMapSchema.java similarity index 94% rename from java-plan3/src/org/msgpack/schema/IMapSchema.java rename to java/src/org/msgpack/schema/IMapSchema.java index 60b3e8e..3a2f556 100644 --- a/java-plan3/src/org/msgpack/schema/IMapSchema.java +++ b/java/src/org/msgpack/schema/IMapSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/IntSchema.java b/java/src/org/msgpack/schema/IntSchema.java similarity index 97% rename from java-plan3/src/org/msgpack/schema/IntSchema.java rename to java/src/org/msgpack/schema/IntSchema.java index c54c0ae..5a7e281 100644 --- a/java-plan3/src/org/msgpack/schema/IntSchema.java +++ b/java/src/org/msgpack/schema/IntSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/LongSchema.java b/java/src/org/msgpack/schema/LongSchema.java similarity index 97% rename from java-plan3/src/org/msgpack/schema/LongSchema.java rename to java/src/org/msgpack/schema/LongSchema.java index ccf3043..83a30e3 100644 --- a/java-plan3/src/org/msgpack/schema/LongSchema.java +++ b/java/src/org/msgpack/schema/LongSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/MapSchema.java b/java/src/org/msgpack/schema/MapSchema.java similarity index 97% rename from java-plan3/src/org/msgpack/schema/MapSchema.java rename to java/src/org/msgpack/schema/MapSchema.java index 71629b0..ba75993 100644 --- a/java-plan3/src/org/msgpack/schema/MapSchema.java +++ b/java/src/org/msgpack/schema/MapSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -88,6 +88,7 @@ public class MapSchema extends Schema implements IMapSchema { } @Override + @SuppressWarnings("unchecked") public Object createFromMap(Object[] obj) { HashMap m = new HashMap(obj.length / 2); int i = 0; diff --git a/java-plan3/src/org/msgpack/schema/RawSchema.java b/java/src/org/msgpack/schema/RawSchema.java similarity index 98% rename from java-plan3/src/org/msgpack/schema/RawSchema.java rename to java/src/org/msgpack/schema/RawSchema.java index 582f766..f621e4c 100644 --- a/java-plan3/src/org/msgpack/schema/RawSchema.java +++ b/java/src/org/msgpack/schema/RawSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java b/java/src/org/msgpack/schema/ReflectionClassSchema.java similarity index 100% rename from java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java rename to java/src/org/msgpack/schema/ReflectionClassSchema.java diff --git a/java-plan3/src/org/msgpack/schema/SSchemaParser.java b/java/src/org/msgpack/schema/SSchemaParser.java similarity index 99% rename from java-plan3/src/org/msgpack/schema/SSchemaParser.java rename to java/src/org/msgpack/schema/SSchemaParser.java index c6bbc77..4ae8a4b 100644 --- a/java-plan3/src/org/msgpack/schema/SSchemaParser.java +++ b/java/src/org/msgpack/schema/SSchemaParser.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/ShortSchema.java b/java/src/org/msgpack/schema/ShortSchema.java similarity index 97% rename from java-plan3/src/org/msgpack/schema/ShortSchema.java rename to java/src/org/msgpack/schema/ShortSchema.java index 089a024..f32ab41 100644 --- a/java-plan3/src/org/msgpack/schema/ShortSchema.java +++ b/java/src/org/msgpack/schema/ShortSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/SpecificClassSchema.java b/java/src/org/msgpack/schema/SpecificClassSchema.java similarity index 98% rename from java-plan3/src/org/msgpack/schema/SpecificClassSchema.java rename to java/src/org/msgpack/schema/SpecificClassSchema.java index 81e5e00..30bd9e1 100644 --- a/java-plan3/src/org/msgpack/schema/SpecificClassSchema.java +++ b/java/src/org/msgpack/schema/SpecificClassSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/StringSchema.java b/java/src/org/msgpack/schema/StringSchema.java similarity index 98% rename from java-plan3/src/org/msgpack/schema/StringSchema.java rename to java/src/org/msgpack/schema/StringSchema.java index f5e0bf1..46d515b 100644 --- a/java-plan3/src/org/msgpack/schema/StringSchema.java +++ b/java/src/org/msgpack/schema/StringSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/test/Generate.java b/java/test/Generate.java similarity index 100% rename from java-plan3/test/Generate.java rename to java/test/Generate.java diff --git a/java/test/README b/java/test/README new file mode 100644 index 0000000..4e16454 --- /dev/null +++ b/java/test/README @@ -0,0 +1,7 @@ +#!/bin/sh +svn checkout -r114 http://thrift-protobuf-compare.googlecode.com/svn/trunk/ thrift-protobuf-compare-base +cp -rf thrift-protobuf-compare/tpc thrift-protobuf-compare-base +cp ../dist/msgpack.jar thrift-protobuf-compare-base/tpc/lib/ +cd thrift-protobuf-compare-base/tpc/ +ant compile +./run-benchmark.sh diff --git a/java/test/thrift-protobuf-compare/tpc/src/serializers/BenchmarkRunner.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/BenchmarkRunner.java new file mode 100644 index 0000000..b17dfb2 --- /dev/null +++ b/java/test/thrift-protobuf-compare/tpc/src/serializers/BenchmarkRunner.java @@ -0,0 +1,434 @@ +package serializers; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; + +import serializers.msgpack.MessagePackSpecificSerializer; +import serializers.msgpack.MessagePackIndirectSerializer; +import serializers.msgpack.MessagePackDynamicSerializer; +import serializers.msgpack.MessagePackGenericSerializer; +import serializers.avro.AvroGenericSerializer; +import serializers.avro.specific.AvroSpecificSerializer; +import serializers.kryo.KryoOptimizedSerializer; +import serializers.kryo.KryoSerializer; + +public class BenchmarkRunner +{ + public final static int ITERATIONS = 2000; + public final static int TRIALS = 20; + + /** + * Number of milliseconds to warm up for each operation type for each serializer. Let's + * start with 3 seconds. + */ + final static long WARMUP_MSECS = 3000; + + @SuppressWarnings("unchecked") + private Set _serializers = new LinkedHashSet(); + + public static void main(String... args) throws Exception + { + BenchmarkRunner runner = new BenchmarkRunner(); + + // binary codecs first + runner.addObjectSerializer(new MessagePackSpecificSerializer()); + runner.addObjectSerializer(new MessagePackIndirectSerializer()); + runner.addObjectSerializer(new MessagePackDynamicSerializer()); + runner.addObjectSerializer(new MessagePackGenericSerializer()); + runner.addObjectSerializer(new AvroGenericSerializer()); + runner.addObjectSerializer(new AvroSpecificSerializer()); + runner.addObjectSerializer(new ActiveMQProtobufSerializer()); + runner.addObjectSerializer(new ProtobufSerializer()); + runner.addObjectSerializer(new ThriftSerializer()); + runner.addObjectSerializer(new HessianSerializer()); + runner.addObjectSerializer(new KryoSerializer()); + runner.addObjectSerializer(new KryoOptimizedSerializer()); + + // None of the other serializers use compression, so we'll leave this out. + // runner.addObjectSerializer(new KryoCompressedSerializer()); + + // then language default serializers + runner.addObjectSerializer(new JavaSerializer()); + + runner.addObjectSerializer(new JavaExtSerializer()); + runner.addObjectSerializer(new ScalaSerializer()); + + // then Json + runner.addObjectSerializer(new JsonSerializer()); + runner.addObjectSerializer(new JsonDataBindingSerializer()); + runner.addObjectSerializer(new JsonMarshallerSerializer()); + runner.addObjectSerializer(new ProtostuffJsonSerializer()); + runner.addObjectSerializer(new ProtostuffNumericJsonSerializer()); + // this is pretty slow; so slow that it's almost not worth keeping but: + runner.addObjectSerializer(new GsonSerializer()); + + // then xml via stax, textual and binary + runner.addObjectSerializer(new StaxSerializer("stax/woodstox", + new com.ctc.wstx.stax.WstxInputFactory(), + new com.ctc.wstx.stax.WstxOutputFactory())); + runner.addObjectSerializer(new StaxSerializer("stax/aalto", + new com.fasterxml.aalto.stax.InputFactoryImpl(), + new com.fasterxml.aalto.stax.OutputFactoryImpl())); + + runner.addObjectSerializer(new StaxSerializer("binaryxml/FI", + new com.sun.xml.fastinfoset.stax.factory.StAXInputFactory(), + new com.sun.xml.fastinfoset.stax.factory.StAXOutputFactory())); + + // No point in running all 4 variants: let's just use fastest one: + //runner.addObjectSerializer(new XStreamSerializer("xstream (xpp)", false, null, null)); + //runner.addObjectSerializer(new XStreamSerializer("xstream (xpp with conv)", true, null, null)); + //runner.addObjectSerializer(new XStreamSerializer("xstream (stax)", false, new com.ctc.wstx.stax.WstxInputFactory(), new com.ctc.wstx.stax.WstxOutputFactory())); + runner.addObjectSerializer(new XStreamSerializer("xstream (stax with conv)", + true, + new com.ctc.wstx.stax.WstxInputFactory(), + new com.ctc.wstx.stax.WstxOutputFactory())); + runner.addObjectSerializer(new JavolutionXMLFormatSerializer()); + + runner.addObjectSerializer(new SbinarySerializer()); + // broken? Does not correctly round-trip: + // runner.addObjectSerializer(new YamlSerializer()); + + System.out.println("Starting"); + + runner.start(); + } + + @SuppressWarnings("unchecked") + private void addObjectSerializer(ObjectSerializer serializer) + { + _serializers.add(serializer); + } + + private double createObjects(ObjectSerializer serializer, int iterations) throws Exception + { + long start = System.nanoTime(); + for (int i = 0; i < iterations; i++) + { + serializer.create(); + } + return iterationTime(System.nanoTime() - start, iterations); + } + + private double iterationTime(long delta, int iterations) + { + return (double) delta / (double) (iterations); + } + + private double serializeDifferentObjects(ObjectSerializer serializer, int iterations) throws Exception + { + long start = System.nanoTime(); + for (int i = 0; i < iterations; i++) + { + T obj = serializer.create(); + serializer.serialize(obj); + } + return iterationTime(System.nanoTime()-start, iterations); + } + + + private double serializeSameObject(ObjectSerializer serializer, int iterations) throws Exception + { + // let's reuse same instance to reduce overhead + T obj = serializer.create(); + long delta = 0; + for (int i = 0; i < iterations; i++) + { + long start = System.nanoTime(); + serializer.serialize(obj); + delta += System.nanoTime() - start; + if (i % 1000 == 0) + doGc(); + } + return iterationTime(delta, iterations); + } + + private double deserializeNoFieldAccess(ObjectSerializer serializer, int iterations) throws Exception + { + byte[] array = serializer.serialize(serializer.create()); + long start = System.nanoTime(); + T result = null; + for (int i = 0; i < iterations; i++) + { + result = serializer.deserialize(array); + } + return iterationTime(System.nanoTime()-start, iterations); + } + + private double deserializeAndCheckAllFields(CheckingObjectSerializer serializer, int iterations) throws Exception + { + byte[] array = serializer.serialize(serializer.create()); + long delta = 0; + for (int i = 0; i < iterations; i++) + { + long start = System.nanoTime(); + T obj = serializer.deserialize(array); + serializer.checkAllFields(obj); + delta += System.nanoTime() - start; + } + return iterationTime(delta, iterations); + } + + private double deserializeAndCheckMediaField(CheckingObjectSerializer serializer, int iterations) throws Exception + { + byte[] array = serializer.serialize(serializer.create()); + long delta = 0; + for (int i = 0; i < iterations; i++) + { + long start = System.nanoTime(); + T obj = serializer.deserialize(array); + serializer.checkMediaField(obj); + delta += System.nanoTime() - start; + } + return iterationTime(delta, iterations); + } + + /** + * JVM is not required to honor GC requests, but adding bit of sleep around request is + * most likely to give it a chance to do it. + */ + private void doGc() + { + try { + Thread.sleep(50L); + } catch (InterruptedException ie) { } + System.gc(); + try { // longer sleep afterwards (not needed by GC, but may help with scheduling) + Thread.sleep(200L); + } catch (InterruptedException ie) { } + } + + enum measurements + { + timeCreate, timeSerializeDifferentObjects, timeSerializeSameObject, timeDeserializeNoFieldAccess, timeDeserializeAndCheckMediaField, timeDeserializeAndCheckAllFields, totalTime, length + } + + @SuppressWarnings("unchecked") + private void start() throws Exception + { + System.out.printf("%-24s, %15s, %15s, %15s, %15s, %15s, %15s, %15s, %10s\n", + " ", + "Object create", + "Serialize", + "/w Same Object", + "Deserialize", + "and Check Media", + "and Check All", + "Total Time", + "Serialized Size"); + EnumMap> values = new EnumMap>(measurements.class); + for (measurements m : measurements.values()) + values.put(m, new HashMap()); + + for (ObjectSerializer serializer : _serializers) + { + /* + * Should only warm things for the serializer that we test next: HotSpot JIT will + * otherwise spent most of its time optimizing slower ones... Use + * -XX:CompileThreshold=1 to hint the JIT to start immediately + * + * Actually: 1 is often not a good value -- threshold is the number + * of samples needed to trigger inlining, and there's no point in + * inlining everything. Default value is in thousands, so lowering + * it to, say, 1000 is usually better. + */ + warmCreation(serializer); + doGc(); + double timeCreate = Double.MAX_VALUE; + // do more iteration for object creation because of its short time + for (int i = 0; i < TRIALS; i++) + timeCreate = Math.min(timeCreate, createObjects(serializer, ITERATIONS * 100)); + + warmSerialization(serializer); + + // actually: let's verify serializer actually works now: + checkCorrectness(serializer); + + doGc(); + double timeSerializeDifferentObjects = Double.MAX_VALUE; + for (int i = 0; i < TRIALS; i++) + timeSerializeDifferentObjects = Math.min(timeSerializeDifferentObjects, serializeDifferentObjects(serializer, ITERATIONS)); + + doGc(); + double timeSerializeSameObject = Double.MAX_VALUE; + for (int i = 0; i < TRIALS; i++) + timeSerializeSameObject = Math.min(timeSerializeSameObject, serializeSameObject(serializer, ITERATIONS)); + + warmDeserialization(serializer); + + doGc(); + double timeDeserializeNoFieldAccess = Double.MAX_VALUE; + for (int i = 0; i < TRIALS; i++) + timeDeserializeNoFieldAccess = Math.min(timeDeserializeNoFieldAccess, deserializeNoFieldAccess(serializer, ITERATIONS)); + + double timeDeserializeAndCheckAllFields = Double.NaN; + double timeDeserializeAndCheckMediaField = Double.NaN; + + double totalTime = timeSerializeDifferentObjects + timeDeserializeNoFieldAccess; + + if( serializer instanceof CheckingObjectSerializer) { + CheckingObjectSerializer checkingSerializer = (CheckingObjectSerializer)serializer; + + timeDeserializeAndCheckMediaField = Double.MAX_VALUE; + doGc(); + for (int i = 0; i < TRIALS; i++) + timeDeserializeAndCheckMediaField = Math.min(timeDeserializeAndCheckMediaField, deserializeAndCheckMediaField(checkingSerializer, ITERATIONS)); + + timeDeserializeAndCheckAllFields = Double.MAX_VALUE; + doGc(); + for (int i = 0; i < TRIALS; i++) + timeDeserializeAndCheckAllFields = Math.min(timeDeserializeAndCheckAllFields, deserializeAndCheckAllFields(checkingSerializer, ITERATIONS)); + + totalTime = timeSerializeDifferentObjects + timeDeserializeAndCheckAllFields; + } + + + byte[] array = serializer.serialize(serializer.create()); + System.out.printf("%-24s, %15.5f, %15.5f, %15.5f, %15.5f, %15.5f, %15.5f, %15.5f, %10d\n", + serializer.getName(), + timeCreate, + timeSerializeDifferentObjects, + timeSerializeSameObject, + timeDeserializeNoFieldAccess, + timeDeserializeAndCheckMediaField, + timeDeserializeAndCheckAllFields, + totalTime, + array.length); + + addValue(values, serializer.getName(), timeCreate, timeSerializeDifferentObjects, timeSerializeSameObject, + timeDeserializeNoFieldAccess, timeDeserializeAndCheckMediaField, timeDeserializeAndCheckAllFields, totalTime, array.length); + } + printImages(values); + } + + /** + * Method that tries to validate correctness of serializer, using + * round-trip (construct, serializer, deserialize; compare objects + * after steps 1 and 3). + * Currently only done for StdMediaDeserializer... + */ + private void checkCorrectness(ObjectSerializer serializer) + throws Exception + { + Object input = serializer.create(); + byte[] array = serializer.serialize(input); + Object output = serializer.deserialize(array); + + if (!input.equals(output)) { + /* Should throw an exception; but for now (that we have a few + * failures) let's just whine... + */ + String msg = "serializer '"+serializer.getName()+"' failed round-trip test (ser+deser produces Object different from input), input="+input+", output="+output; + //throw new Exception("Error: "+msg); + System.err.println("WARN: "+msg); + } + } + + private void printImages(EnumMap> values) + { + for (measurements m : values.keySet()) { + Map map = values.get(m); + ArrayList list = new ArrayList(map.entrySet()); + Collections.sort(list, new Comparator() { + public int compare (Entry o1, Entry o2) { + double diff = (Double)o1.getValue() - (Double)o2.getValue(); + return diff > 0 ? 1 : (diff < 0 ? -1 : 0); + } + }); + LinkedHashMap sortedMap = new LinkedHashMap(); + for (Entry entry : list) { + if( !entry.getValue().isNaN() ) { + sortedMap.put(entry.getKey(), entry.getValue()); + } + } + printImage(sortedMap, m); + } + } + + private void printImage(Map map, measurements m) + { + StringBuilder valSb = new StringBuilder(); + String names = ""; + double max = Double.MIN_NORMAL; + for (Entry entry : map.entrySet()) + { + valSb.append(entry.getValue()).append(','); + max = Math.max(max, entry.getValue()); + names = entry.getKey() + '|' + names; + } + + int height = Math.min(30+map.size()*20, 430); + double scale = max * 1.1; + System.out.println(""); + + } + + private void addValue(EnumMap> values, + String name, + double timeCreate, + double timeSerializeDifferentObjects, + double timeSerializeSameObject, + double timeDeserializeNoFieldAccess, + double timeDeserializeAndCheckMediaField, + double timeDeserializeAndCheckAllFields, + double totalTime, + double length) + { + + values.get(measurements.timeCreate).put(name, timeCreate); + values.get(measurements.timeSerializeDifferentObjects).put(name, timeSerializeDifferentObjects); + values.get(measurements.timeSerializeSameObject).put(name, timeSerializeSameObject); + values.get(measurements.timeDeserializeNoFieldAccess).put(name, timeDeserializeNoFieldAccess); + values.get(measurements.timeDeserializeAndCheckMediaField).put(name, timeDeserializeAndCheckMediaField); + values.get(measurements.timeDeserializeAndCheckAllFields).put(name, timeDeserializeAndCheckAllFields); + values.get(measurements.totalTime).put(name, totalTime); + values.get(measurements.length).put(name, length); + } + + private void warmCreation(ObjectSerializer serializer) throws Exception + { + // Instead of fixed counts, let's try to prime by running for N seconds + long endTime = System.currentTimeMillis() + WARMUP_MSECS; + do + { + createObjects(serializer, 1); + } + while (System.currentTimeMillis() < endTime); + } + + private void warmSerialization(ObjectSerializer serializer) throws Exception + { + // Instead of fixed counts, let's try to prime by running for N seconds + long endTime = System.currentTimeMillis() + WARMUP_MSECS; + do + { + serializeDifferentObjects(serializer, 1); + } + while (System.currentTimeMillis() < endTime); + } + + private void warmDeserialization(ObjectSerializer serializer) throws Exception + { + // Instead of fixed counts, let's try to prime by running for N seconds + long endTime = System.currentTimeMillis() + WARMUP_MSECS; + do + { + deserializeNoFieldAccess(serializer, 1); + } + while (System.currentTimeMillis() < endTime); + } +} diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java similarity index 100% rename from java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java rename to java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java diff --git a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs similarity index 100% rename from java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs rename to java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java similarity index 100% rename from java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java rename to java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java similarity index 100% rename from java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java rename to java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java similarity index 100% rename from java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java rename to java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java similarity index 100% rename from java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java rename to java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java From 55cfbf378e5fcf91d0dffe73f9cab3f3f7414233 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 14 Jan 2010 21:20:32 +0900 Subject: [PATCH 0346/1172] cpp: add msgpack/type/tr1/unordered_{map,set}.hpp --- configure.in | 2 +- cpp/Makefile.am | 4 +- cpp/type/set.hpp | 25 ++++++++++ cpp/type/tr1/unordered_map.hpp | 85 ++++++++++++++++++++++++++++++++++ cpp/type/tr1/unordered_set.hpp | 80 ++++++++++++++++++++++++++++++++ 5 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 cpp/type/tr1/unordered_map.hpp create mode 100644 cpp/type/tr1/unordered_set.hpp diff --git a/configure.in b/configure.in index 76bd7e4..32d906e 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.9) +AM_INIT_AUTOMAKE(msgpack, 0.3.10) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 42d6d2a..0923362 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -25,7 +25,9 @@ nobase_include_HEADERS = \ msgpack/type/string.hpp \ msgpack/type/vector.hpp \ msgpack/type/tuple.hpp \ - msgpack/type/define.hpp + msgpack/type/define.hpp \ + msgpack/type/tr1/unordered_map.hpp \ + msgpack/type/tr1/unordered_set.hpp libmsgpack_la_LIBADD = -L../c -lmsgpackc diff --git a/cpp/type/set.hpp b/cpp/type/set.hpp index 11db2b3..f2c5bfb 100644 --- a/cpp/type/set.hpp +++ b/cpp/type/set.hpp @@ -49,6 +49,31 @@ inline packer& operator<< (packer& o, const std::set& v) } +template +inline std::multiset& operator>> (object o, std::multiset& v) +{ + if(o.type != type::ARRAY) { throw type_error(); } + object* p = o.via.array.ptr + o.via.array.size; + object* const pbegin = o.via.array.ptr; + while(p > pbegin) { + --p; + v.insert(p->as()); + } + return v; +} + +template +inline packer& operator<< (packer& o, const std::multiset& v) +{ + o.pack_array(v.size()); + for(typename std::multiset::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(*it); + } + return o; +} + + } // namespace msgpack #endif /* msgpack/type/set.hpp */ diff --git a/cpp/type/tr1/unordered_map.hpp b/cpp/type/tr1/unordered_map.hpp new file mode 100644 index 0000000..1996cfd --- /dev/null +++ b/cpp/type/tr1/unordered_map.hpp @@ -0,0 +1,85 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_TR1_UNORDERED_MAP_HPP__ +#define MSGPACK_TYPE_TR1_UNORDERED_MAP_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { + + +template +inline std::tr1::unordered_map operator>> (object o, std::tr1::unordered_map& v) +{ + if(o.type != type::MAP) { throw type_error(); } + object_kv* p(o.via.map.ptr); + object_kv* const pend(o.via.map.ptr + o.via.map.size); + for(; p != pend; ++p) { + K key; + p->key.convert(&key); + p->val.convert(&v[key]); + } + return v; +} + +template +inline packer& operator<< (packer& o, const std::tr1::unordered_map& v) +{ + o.pack_map(v.size()); + for(typename std::tr1::unordered_map::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(it->first); + o.pack(it->second); + } + return o; +} + + +template +inline std::tr1::unordered_multimap operator>> (object o, std::tr1::unordered_multimap& v) +{ + if(o.type != type::MAP) { throw type_error(); } + object_kv* p(o.via.map.ptr); + object_kv* const pend(o.via.map.ptr + o.via.map.size); + for(; p != pend; ++p) { + std::pair value; + p->key.convert(&value.first); + p->val.convert(&value.second); + v.insert(value); + } + return v; +} + +template +inline packer& operator<< (packer& o, const std::tr1::unordered_multimap& v) +{ + o.pack_map(v.size()); + for(typename std::tr1::unordered_multimap::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(it->first); + o.pack(it->second); + } + return o; +} + + +} // namespace msgpack + +#endif /* msgpack/type/map.hpp */ + diff --git a/cpp/type/tr1/unordered_set.hpp b/cpp/type/tr1/unordered_set.hpp new file mode 100644 index 0000000..eb127b5 --- /dev/null +++ b/cpp/type/tr1/unordered_set.hpp @@ -0,0 +1,80 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_TR1_UNORDERED_SET_HPP__ +#define MSGPACK_TYPE_TR1_UNORDERED_SET_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { + + +template +inline std::tr1::unordered_set& operator>> (object o, std::tr1::unordered_set& v) +{ + if(o.type != type::ARRAY) { throw type_error(); } + object* p = o.via.array.ptr + o.via.array.size; + object* const pbegin = o.via.array.ptr; + while(p > pbegin) { + --p; + v.insert(p->as()); + } + return v; +} + +template +inline packer& operator<< (packer& o, const std::tr1::unordered_set& v) +{ + o.pack_array(v.size()); + for(typename std::tr1::unordered_set::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(*it); + } + return o; +} + + +template +inline std::tr1::unordered_multiset& operator>> (object o, std::tr1::unordered_multiset& v) +{ + if(o.type != type::ARRAY) { throw type_error(); } + object* p = o.via.array.ptr + o.via.array.size; + object* const pbegin = o.via.array.ptr; + while(p > pbegin) { + --p; + v.insert(p->as()); + } + return v; +} + +template +inline packer& operator<< (packer& o, const std::tr1::unordered_multiset& v) +{ + o.pack_array(v.size()); + for(typename std::tr1::unordered_multiset::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(*it); + } + return o; +} + + +} // namespace msgpack + +#endif /* msgpack/type/set.hpp */ + From 8d365458d56cc3ac9ce74d8cc1470ede30465522 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 15 Jan 2010 05:27:44 +0900 Subject: [PATCH 0347/1172] c,cpp: optimize msgpack_zone: variable-length array of chunk -> list of chunk --- c/Makefile.am | 2 +- c/zone.c | 153 +++++++++++++++++++++-------------------------- c/zone.h | 35 +++++------ cpp/Makefile.am | 2 +- cpp/zone.hpp.erb | 5 +- 5 files changed, 88 insertions(+), 109 deletions(-) diff --git a/c/Makefile.am b/c/Makefile.am index 0a52446..23995a5 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -16,7 +16,7 @@ nobase_include_HEADERS = \ msgpack/zone.h # -version-info CURRENT:REVISION:AGE -libmsgpackc_la_LDFLAGS = -version-info 1:0:0 +libmsgpackc_la_LDFLAGS = -version-info 2:0:0 check_PROGRAMS = \ msgpackc_test diff --git a/c/zone.c b/c/zone.c index 79e90dc..2d8bc58 100644 --- a/c/zone.c +++ b/c/zone.c @@ -19,83 +19,61 @@ #include #include -static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_size) -{ - // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ - const size_t nfirst = (sizeof(msgpack_zone_chunk) < 72/2) ? - 72 / sizeof(msgpack_zone_chunk) : 8; +struct msgpack_zone_chunk { + struct msgpack_zone_chunk* next; + /* data ... */ +}; - msgpack_zone_chunk* array = (msgpack_zone_chunk*)malloc( - sizeof(msgpack_zone_chunk) * nfirst); - if(array == NULL) { +static inline bool init_chunk_list(msgpack_zone_chunk_list* cl, size_t chunk_size) +{ + msgpack_zone_chunk* chunk = (msgpack_zone_chunk*)malloc( + sizeof(msgpack_zone_chunk) + chunk_size); + if(chunk == NULL) { return false; } - const size_t sz = chunk_size; - - char* ptr = (char*)malloc(sz); - if(ptr == NULL) { - free(array); - return NULL; - } - - ca->tail = array; - ca->end = array + nfirst; - ca->array = array; - - array[0].free = sz; - array[0].ptr = ptr; - array[0].alloc = ptr; + cl->head = chunk; + cl->free = chunk_size; + cl->ptr = ((char*)chunk) + sizeof(msgpack_zone_chunk); + chunk->next = NULL; return true; } -static inline void destroy_chunk_array(msgpack_zone_chunk_array* ca) +static inline void destroy_chunk_list(msgpack_zone_chunk_list* cl) { - msgpack_zone_chunk* chunk = ca->array; - for(; chunk != ca->tail+1; ++chunk) { - free(chunk->alloc); + msgpack_zone_chunk* c = cl->head; + while(true) { + msgpack_zone_chunk* n = c->next; + free(c); + if(n != NULL) { + c = n; + } else { + break; + } } - - free(ca->array); } -static inline void clear_chunk_array(msgpack_zone_chunk_array* ca) +static inline void clear_chunk_list(msgpack_zone_chunk_list* cl, size_t chunk_size) { - msgpack_zone_chunk* chunk = ca->array + 1; - for(; chunk != ca->tail+1; ++chunk) { - free(chunk->alloc); + msgpack_zone_chunk* c = cl->head; + while(true) { + msgpack_zone_chunk* n = c->next; + if(n != NULL) { + free(c); + c = n; + } else { + break; + } } - - ca->tail = ca->array; - - ca->array[0].free += ca->array[0].ptr - (char*)ca->array[0].alloc; - ca->array[0].ptr = (char*)ca->array[0].alloc; + cl->head->next = NULL; + cl->free = chunk_size; + cl->ptr = ((char*)cl->head) + sizeof(msgpack_zone_chunk); } void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) { - msgpack_zone_chunk_array* const ca = &zone->chunk_array; - - msgpack_zone_chunk* chunk = ++ca->tail; - - if(chunk == ca->end) { - // ca->arrayã«ç©ºããŒãªã„ - // ca->arrayã‚’æ‹¡å¼µã™ã‚‹ - - const size_t nused = ca->end - ca->array; - const size_t nnext = (ca->end - ca->array) * 2; - - chunk = (msgpack_zone_chunk*)realloc(ca->array, - sizeof(msgpack_zone_chunk) * nnext); - if(chunk == NULL) { - return NULL; - } - - ca->array = chunk; - ca->end = chunk + nnext; - chunk = ca->tail = chunk + nused; - } + msgpack_zone_chunk_list* const cl = &zone->chunk_list; size_t sz = zone->chunk_size; @@ -103,14 +81,15 @@ void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) sz *= 2; } - char* ptr = (char*)malloc(sz); - if(ptr == NULL) { - return NULL; - } + msgpack_zone_chunk* chunk = (msgpack_zone_chunk*)malloc( + sizeof(msgpack_zone_chunk) + sz); - chunk->free = sz - size; - chunk->ptr = ptr + size; - chunk->alloc = ptr; + char* ptr = ((char*)chunk) + sizeof(msgpack_zone_chunk); + + chunk->next = cl->head; + cl->head = chunk; + cl->free = sz - size; + cl->ptr = ptr + size; return ptr; } @@ -185,19 +164,30 @@ bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, bool msgpack_zone_is_empty(msgpack_zone* zone) { - msgpack_zone_chunk_array* const ca = &zone->chunk_array; + msgpack_zone_chunk_list* const cl = &zone->chunk_list; msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; - return ca->array[0].ptr == ca->array[0].alloc && - ca->tail == ca->array && + return cl->free == zone->chunk_size && cl->head->next == NULL && fa->tail == fa->array; } +void msgpack_zone_destroy(msgpack_zone* zone) +{ + destroy_finalizer_array(&zone->finalizer_array); + destroy_chunk_list(&zone->chunk_list); +} + +void msgpack_zone_clear(msgpack_zone* zone) +{ + clear_finalizer_array(&zone->finalizer_array); + clear_chunk_list(&zone->chunk_list, zone->chunk_size); +} + bool msgpack_zone_init(msgpack_zone* zone, size_t chunk_size) { zone->chunk_size = chunk_size; - if(!init_chunk_array(&zone->chunk_array, chunk_size)) { + if(!init_chunk_list(&zone->chunk_list, chunk_size)) { return false; } @@ -206,30 +196,23 @@ bool msgpack_zone_init(msgpack_zone* zone, size_t chunk_size) return true; } -void msgpack_zone_destroy(msgpack_zone* zone) -{ - destroy_finalizer_array(&zone->finalizer_array); - destroy_chunk_array(&zone->chunk_array); -} - -void msgpack_zone_clear(msgpack_zone* zone) -{ - clear_finalizer_array(&zone->finalizer_array); - clear_chunk_array(&zone->chunk_array); -} - msgpack_zone* msgpack_zone_new(size_t chunk_size) { - msgpack_zone* zone = (msgpack_zone*)malloc(sizeof(msgpack_zone)); + msgpack_zone* zone = (msgpack_zone*)malloc( + sizeof(msgpack_zone) + chunk_size); if(zone == NULL) { return NULL; } - if(!msgpack_zone_init(zone, chunk_size)) { + zone->chunk_size = chunk_size; + + if(!init_chunk_list(&zone->chunk_list, chunk_size)) { free(zone); - return NULL; + return false; } + init_finalizer_array(&zone->finalizer_array); + return zone; } diff --git a/c/zone.h b/c/zone.h index 79b51f8..e07e762 100644 --- a/c/zone.h +++ b/c/zone.h @@ -26,37 +26,34 @@ extern "C" { #endif -typedef struct msgpack_zone_chunk { - size_t free; - char* ptr; - void* alloc; -} msgpack_zone_chunk; - typedef struct msgpack_zone_finalizer { void (*func)(void* data); void* data; } msgpack_zone_finalizer; -typedef struct msgpack_zone_chunk_array { - msgpack_zone_chunk* tail; - msgpack_zone_chunk* end; - msgpack_zone_chunk* array; -} msgpack_zone_chunk_array; - typedef struct msgpack_zone_finalizer_array { msgpack_zone_finalizer* tail; msgpack_zone_finalizer* end; msgpack_zone_finalizer* array; } msgpack_zone_finalizer_array; +struct msgpack_zone_chunk; +typedef struct msgpack_zone_chunk msgpack_zone_chunk; + +typedef struct msgpack_zone_chunk_list { + size_t free; + char* ptr; + msgpack_zone_chunk* head; +} msgpack_zone_chunk_list; + typedef struct msgpack_zone { - msgpack_zone_chunk_array chunk_array; + msgpack_zone_chunk_list chunk_list; msgpack_zone_finalizer_array finalizer_array; size_t chunk_size; } msgpack_zone; #ifndef MSGPACK_ZONE_CHUNK_SIZE -#define MSGPACK_ZONE_CHUNK_SIZE 2048 +#define MSGPACK_ZONE_CHUNK_SIZE 8192 #endif bool msgpack_zone_init(msgpack_zone* zone, size_t chunk_size); @@ -85,15 +82,15 @@ void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size); void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size) { - msgpack_zone_chunk* chunk = zone->chunk_array.tail; + msgpack_zone_chunk_list* cl = &zone->chunk_list; - if(chunk->free < size) { + if(zone->chunk_list.free < size) { return msgpack_zone_malloc_expand(zone, size); } - char* ptr = chunk->ptr; - chunk->ptr += size; - chunk->free -= size; + char* ptr = cl->ptr; + cl->free -= size; + cl->ptr += size; return ptr; } diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 0923362..99f4a32 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -32,7 +32,7 @@ nobase_include_HEADERS = \ libmsgpack_la_LIBADD = -L../c -lmsgpackc # -version-info CURRENT:REVISION:AGE -libmsgpack_la_LDFLAGS = -version-info 1:0:0 +libmsgpack_la_LDFLAGS = -version-info 2:0:0 check_PROGRAMS = \ msgpack_test diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index f1e4624..48988ab 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -120,9 +120,8 @@ void zone::object_destructor(void* obj) inline void zone::undo_malloc(size_t size) { - msgpack_zone_chunk* chunk = base::chunk_array.tail; - chunk->ptr -= size; - chunk->free += size; + base::chunk_list.ptr -= size; + base::chunk_list.free += size; } <%0.upto(GENERATION_LIMIT) {|i|%> From f145129f6ec91a13fbec99d4dc5b76654a951f79 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 15 Jan 2010 05:29:41 +0900 Subject: [PATCH 0348/1172] c,cpp: optimize msgpack_vrefbuffer --- c/vrefbuffer.c | 112 +++++++++++++++++++++++++++++++++---------------- c/vrefbuffer.h | 21 ++++++---- 2 files changed, 88 insertions(+), 45 deletions(-) diff --git a/c/vrefbuffer.c b/c/vrefbuffer.c index 2bf97af..b970ece 100644 --- a/c/vrefbuffer.c +++ b/c/vrefbuffer.c @@ -19,15 +19,14 @@ #include #include +struct msgpack_vrefbuffer_chunk { + struct msgpack_vrefbuffer_chunk* next; + /* data ... */ +}; + bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, size_t ref_size, size_t chunk_size) { - if(chunk_size < sizeof(msgpack_vrefbuffer_chunk)+72) { - chunk_size = 72; - } else { - chunk_size -= sizeof(msgpack_vrefbuffer_chunk); - } - vbuf->chunk_size = chunk_size; vbuf->ref_size = ref_size; @@ -45,26 +44,30 @@ bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, vbuf->end = array + nfirst; vbuf->array = array; - vbuf->chunk = (msgpack_vrefbuffer_chunk*)malloc( - chunk_size + sizeof(msgpack_vrefbuffer_chunk)); - if(vbuf->chunk == NULL) { + msgpack_vrefbuffer_chunk* chunk = (msgpack_vrefbuffer_chunk*)malloc( + sizeof(msgpack_vrefbuffer_chunk) + chunk_size); + if(chunk == NULL) { free(array); return false; } - vbuf->chunk->next = NULL; - vbuf->chunk->free = chunk_size; + msgpack_vrefbuffer_inner_buffer* const ib = &vbuf->inner_buffer; + + ib->free = chunk_size; + ib->ptr = ((char*)chunk) + sizeof(msgpack_vrefbuffer_chunk); + ib->head = chunk; + chunk->next = NULL; return true; } void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf) { - msgpack_vrefbuffer_chunk* c = vbuf->chunk; + msgpack_vrefbuffer_chunk* c = vbuf->inner_buffer.head; while(true) { msgpack_vrefbuffer_chunk* n = c->next; free(c); - if(n) { + if(n != NULL) { c = n; } else { break; @@ -101,28 +104,30 @@ int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, const char* buf, unsigned int len) { - msgpack_vrefbuffer_chunk* chunk = vbuf->chunk; - size_t cur_size = vbuf->chunk_size; + msgpack_vrefbuffer_inner_buffer* const ib = &vbuf->inner_buffer; - if(chunk->free < len) { - cur_size = (cur_size > len) ? cur_size : len; + if(ib->free < len) { + size_t sz = vbuf->chunk_size; + if(sz < len) { + sz = len; + } - chunk = (msgpack_vrefbuffer_chunk*)malloc( - cur_size + sizeof(msgpack_vrefbuffer_chunk)); + msgpack_vrefbuffer_chunk* chunk = (msgpack_vrefbuffer_chunk*)malloc( + sizeof(msgpack_vrefbuffer_chunk) + sz); if(chunk == NULL) { return -1; } - chunk->free = cur_size; - chunk->next = vbuf->chunk; - vbuf->chunk = chunk; + chunk->next = ib->head; + ib->head = chunk; + ib->free = sz; + ib->ptr = ((char*)chunk) + sizeof(msgpack_vrefbuffer_chunk); } - char* m = ((char*)chunk) + sizeof(msgpack_vrefbuffer_chunk) - + (cur_size - chunk->free); - + char* m = ib->ptr; memcpy(m, buf, len); - chunk->free -= len; + ib->free -= len; + ib->ptr += len; if(vbuf->tail != vbuf->array && m == (const char*)((vbuf->tail-1)->iov_base) + (vbuf->tail-1)->iov_len) { @@ -135,28 +140,63 @@ int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, int msgpack_vrefbuffer_migrate(msgpack_vrefbuffer* vbuf, msgpack_vrefbuffer* to) { - const size_t tosize = to->tail - to->array; - if(vbuf->tail + tosize < vbuf->end) { - const size_t nused = vbuf->tail - vbuf->array; - const size_t nsize = vbuf->end - vbuf->array; + size_t sz = vbuf->chunk_size; + + msgpack_vrefbuffer_chunk* empty = (msgpack_vrefbuffer_chunk*)malloc( + sizeof(msgpack_vrefbuffer_chunk) + sz); + if(empty == NULL) { + return -1; + } + + empty->next = NULL; + + + const size_t nused = vbuf->tail - vbuf->array; + if(to->tail + nused < vbuf->end) { + const size_t tosize = to->tail - to->array; const size_t reqsize = nused + tosize; - size_t nnext = nsize * 2; + size_t nnext = (to->end - to->array) * 2; while(nnext < reqsize) { nnext *= 2; } struct iovec* nvec = (struct iovec*)realloc( - vbuf->array, sizeof(struct iovec)*nnext); + to->array, sizeof(struct iovec)*nnext); if(nvec == NULL) { + free(empty); return -1; } - vbuf->array = nvec; - vbuf->end = nvec + nnext; - vbuf->tail = nvec + nused; + to->array = nvec; + to->end = nvec + nnext; + to->tail = nvec + tosize; } - memcpy(vbuf->tail, vbuf->array, sizeof(struct iovec)*tosize); + memcpy(to->tail, vbuf->array, sizeof(struct iovec)*nused); + + to->tail += nused; + vbuf->tail = vbuf->array; + + + msgpack_vrefbuffer_inner_buffer* const ib = &vbuf->inner_buffer; + msgpack_vrefbuffer_inner_buffer* const toib = &to->inner_buffer; + + msgpack_vrefbuffer_chunk* last = ib->head; + while(last->next != NULL) { + last = last->next; + } + last->next = toib->head; + toib->head = ib->head; + + if(toib->free < ib->free) { + toib->free = ib->free; + toib->ptr = ib->ptr; + } + + ib->head = empty; + ib->free = sz; + ib->ptr = ((char*)empty) + sizeof(msgpack_vrefbuffer_chunk); + return 0; } diff --git a/c/vrefbuffer.h b/c/vrefbuffer.h index 9f24f14..ffc52bf 100644 --- a/c/vrefbuffer.h +++ b/c/vrefbuffer.h @@ -40,24 +40,27 @@ extern "C" { #endif #ifndef MSGPACK_VREFBUFFER_CHUNK_SIZE -#define MSGPACK_VREFBUFFER_CHUNK_SIZE 2048 +#define MSGPACK_VREFBUFFER_CHUNK_SIZE 8192 #endif -typedef struct msgpack_vrefbuffer_chunk { +struct msgpack_vrefbuffer_chunk; +typedef struct msgpack_vrefbuffer_chunk msgpack_vrefbuffer_chunk; + +typedef struct msgpack_vrefbuffer_inner_buffer { size_t free; - struct msgpack_vrefbuffer_chunk* next; - /* data ... */ -} msgpack_vrefbuffer_chunk; + char* ptr; + msgpack_vrefbuffer_chunk* head; +} msgpack_vrefbuffer_inner_buffer; typedef struct msgpack_vrefbuffer { - size_t chunk_size; - size_t ref_size; - struct iovec* tail; struct iovec* end; struct iovec* array; - msgpack_vrefbuffer_chunk* chunk; + size_t chunk_size; + size_t ref_size; + + msgpack_vrefbuffer_inner_buffer inner_buffer; } msgpack_vrefbuffer; From 2bf3f1856f23d0759686ab74b8abf7f46841564a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 15 Jan 2010 14:45:52 +0900 Subject: [PATCH 0349/1172] c,cpp: configure.in: version 0.4.0 --- configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 32d906e..3d2247e 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.10) +AM_INIT_AUTOMAKE(msgpack, 0.4.0) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) From 1066bb38a87df13e89d891682e39df2ef9a08d7c Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sat, 16 Jan 2010 02:03:11 +0900 Subject: [PATCH 0350/1172] add tests for multi[map, set] and tr1::unordered_[multi][map, set] --- configure.in | 5 +- cpp/test.cpp | 190 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 3d2247e..99c4352 100644 --- a/configure.in +++ b/configure.in @@ -12,7 +12,6 @@ AC_PROG_CC CFLAGS="-O4 -Wall $CFLAGS -I.." - AC_MSG_CHECKING([if c++ api is enabled]) AC_ARG_ENABLE(cxx, AS_HELP_STRING([--disable-cxx], @@ -30,6 +29,10 @@ AC_PROG_CXX CXXFLAGS="-O4 -Wall $CXXFLAGS -I.. -I../c" +AC_LANG_PUSH([C++]) +AC_CHECK_HEADERS(tr1/unordered_map) +AC_CHECK_HEADERS(tr1/unordered_set) +AC_LANG_POP([C++]) AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no") diff --git a/cpp/test.cpp b/cpp/test.cpp index 4447890..b152e6d 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -10,6 +10,10 @@ #include +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + using namespace std; const unsigned int kLoop = 10000; @@ -227,6 +231,8 @@ TEST(MSGPACK, simple_buffer_false) //----------------------------------------------------------------------------- +// STL + TEST(MSGPACK_STL, simple_buffer_string) { for (unsigned int k = 0; k < kLoop; k++) { @@ -365,6 +371,190 @@ TEST(MSGPACK_STL, simple_buffer_pair) } } +TEST(MSGPACK_STL, simple_buffer_multimap) +{ + for (unsigned int k = 0; k < kLoop; k++) { + multimap val1; + for (unsigned int i = 0; i < kElements; i++) { + int i1 = rand(); + val1.insert(make_pair(i1, rand())); + val1.insert(make_pair(i1, rand())); + } + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); + multimap val2; + obj.convert(&val2); + + vector > v1, v2; + multimap::const_iterator it; + for (it = val1.begin(); it != val1.end(); ++it) + v1.push_back(make_pair(it->first, it->second)); + for (it = val2.begin(); it != val2.end(); ++it) + v2.push_back(make_pair(it->first, it->second)); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_EQ(v1.size(), v2.size()); + sort(v1.begin(), v1.end()); + sort(v2.begin(), v2.end()); + EXPECT_TRUE(v1 == v2); + } +} + +TEST(MSGPACK_STL, simple_buffer_multiset) +{ + for (unsigned int k = 0; k < kLoop; k++) { + multiset val1; + for (unsigned int i = 0; i < kElements; i++) + val1.insert(rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); + multiset val2; + obj.convert(&val2); + + vector v1, v2; + multiset::const_iterator it; + for (it = val1.begin(); it != val1.end(); ++it) + v1.push_back(*it); + for (it = val2.begin(); it != val2.end(); ++it) + v2.push_back(*it); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_EQ(v1.size(), v2.size()); + sort(v1.begin(), v1.end()); + sort(v2.begin(), v2.end()); + EXPECT_TRUE(v1 == v2); + } +} + +// TR1 + +#ifdef HAVE_TR1_UNORDERED_MAP +#include +#include "cpp/type/tr1/unordered_map.hpp" +TEST(MSGPACK_TR1, simple_buffer_unordered_map) +{ + for (unsigned int k = 0; k < kLoop; k++) { + tr1::unordered_map val1; + for (unsigned int i = 0; i < kElements; i++) + val1[rand()] = rand(); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); + tr1::unordered_map val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + tr1::unordered_map::const_iterator it; + for (it = val1.begin(); it != val1.end(); ++it) { + EXPECT_TRUE(val2.find(it->first) != val2.end()); + EXPECT_EQ(it->second, val2.find(it->first)->second); + } + } +} + +TEST(MSGPACK_TR1, simple_buffer_unordered_multimap) +{ + for (unsigned int k = 0; k < kLoop; k++) { + tr1::unordered_multimap val1; + for (unsigned int i = 0; i < kElements; i++) { + int i1 = rand(); + val1.insert(make_pair(i1, rand())); + val1.insert(make_pair(i1, rand())); + } + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); + tr1::unordered_multimap val2; + obj.convert(&val2); + + vector > v1, v2; + tr1::unordered_multimap::const_iterator it; + for (it = val1.begin(); it != val1.end(); ++it) + v1.push_back(make_pair(it->first, it->second)); + for (it = val2.begin(); it != val2.end(); ++it) + v2.push_back(make_pair(it->first, it->second)); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_EQ(v1.size(), v2.size()); + sort(v1.begin(), v1.end()); + sort(v2.begin(), v2.end()); + EXPECT_TRUE(v1 == v2); + } +} +#endif + +#ifdef HAVE_TR1_UNORDERED_SET +#include +#include "cpp/type/tr1/unordered_set.hpp" +TEST(MSGPACK_TR1, simple_buffer_unordered_set) +{ + for (unsigned int k = 0; k < kLoop; k++) { + tr1::unordered_set val1; + for (unsigned int i = 0; i < kElements; i++) + val1.insert(rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); + tr1::unordered_set val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + tr1::unordered_set::const_iterator it; + for (it = val1.begin(); it != val1.end(); ++it) + EXPECT_TRUE(val2.find(*it) != val2.end()); + } +} + +TEST(MSGPACK_TR1, simple_buffer_unordered_multiset) +{ + for (unsigned int k = 0; k < kLoop; k++) { + tr1::unordered_multiset val1; + for (unsigned int i = 0; i < kElements; i++) + val1.insert(rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); + tr1::unordered_multiset val2; + obj.convert(&val2); + + vector v1, v2; + tr1::unordered_multiset::const_iterator it; + for (it = val1.begin(); it != val1.end(); ++it) + v1.push_back(*it); + for (it = val2.begin(); it != val2.end(); ++it) + v2.push_back(*it); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_EQ(v1.size(), v2.size()); + sort(v1.begin(), v1.end()); + sort(v2.begin(), v2.end()); + EXPECT_TRUE(v1 == v2); + } +} +#endif + +// User-Defined Structures + class TestClass { public: From d8212ad6200db16ca49c5c8198ba0955272763c8 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 20 Jan 2010 14:46:54 +0900 Subject: [PATCH 0351/1172] strict-aliasing rule --- msgpack/pack_template.h | 12 ++++++------ msgpack/unpack_template.h | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index de148bf..b314d6d 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -554,19 +554,19 @@ if(sizeof(unsigned long long) == 2) { msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { - union { char buf[4]; uint32_t num; } f; - *((float*)&f.buf) = d; // FIXME + union { float f; uint32_t i; } mem; + mem.f = d; unsigned char buf[5]; - buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(f.num); + buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(mem.i); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { - union { char buf[8]; uint64_t num; } f; - *((double*)&f.buf) = d; // FIXME + union { double f; uint64_t i; } mem; + mem.f = d; unsigned char buf[9]; - buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(f.num); + buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(mem.i); msgpack_pack_append_buffer(x, buf, 9); } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 3328ea3..cf2e74c 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -222,13 +222,13 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c //case CS_ //case CS_ case CS_FLOAT: { - union { uint32_t num; char buf[4]; } f; - f.num = PTR_CAST_32(n); // FIXME - push_fixed_value(_float, *((float*)f.buf)); } + union { uint32_t i; float f; } mem; + mem.i = PTR_CAST_32(n); + push_fixed_value(_float, mem.f); } case CS_DOUBLE: { - union { uint64_t num; char buf[8]; } f; - f.num = PTR_CAST_64(n); // FIXME - push_fixed_value(_double, *((double*)f.buf)); } + union { uint64_t i; double f; } mem; + mem.i = PTR_CAST_64(n); + push_fixed_value(_double, mem.f); } case CS_UINT_8: push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n)); case CS_UINT_16: From 2b72f35c324e3a0623894f9c678bd08341973a32 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 20 Jan 2010 14:49:51 +0900 Subject: [PATCH 0352/1172] configure.in: show error message if __sync_* atomic operations are not supported --- configure.in | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 99c4352..6bf1b48 100644 --- a/configure.in +++ b/configure.in @@ -24,7 +24,7 @@ if test "$enable_cxx" != "no"; then fi fi -# FIXME +# FIXME enable_cxx AC_PROG_CXX CXXFLAGS="-O4 -Wall $CXXFLAGS -I.. -I../c" @@ -34,6 +34,22 @@ AC_CHECK_HEADERS(tr1/unordered_map) AC_CHECK_HEADERS(tr1/unordered_set) AC_LANG_POP([C++]) +AC_CACHE_CHECK([for __sync_* atomic operations], msgpack_cv_atomic_ops, [ + AC_TRY_LINK([ + int atomic_sub(int i) { return __sync_sub_and_fetch(&i, 1); } + int atomic_add(int i) { return __sync_add_and_fetch(&i, 1); } + ], [], msgpack_cv_atomic_ops="yes") + ]) +if test "$msgpack_cv_atomic_ops" != "yes"; then + AC_MSG_ERROR([__sync_* atomic operations are not supported. +Note that gcc < 4.1 is not supported. +If you are using gcc-4.1 and the CPU architecture is x86, try to add +CFLAGS"--march=i686" and CXXFLAGS="-march=i668" options to ./configure as follows: + + $ ./configure CFLAGS="-march=i686" CXXFLAGS="-march=i686" +]) +fi + AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no") AC_PROG_LIBTOOL From ebc0eeac7911891c5794891e48ab8db8605e5ab7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 20 Jan 2010 15:17:33 +0900 Subject: [PATCH 0353/1172] c,cpp: 0.4.1 --- configure.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 6bf1b48..f9f2918 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.4.0) +AM_INIT_AUTOMAKE(msgpack, 0.4.1) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) @@ -29,6 +29,7 @@ AC_PROG_CXX CXXFLAGS="-O4 -Wall $CXXFLAGS -I.. -I../c" +# FIXME enable_cxx AC_LANG_PUSH([C++]) AC_CHECK_HEADERS(tr1/unordered_map) AC_CHECK_HEADERS(tr1/unordered_set) From c69092e110b82f4a6b37759a37cd43dbe8e486ed Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 21 Jan 2010 17:25:32 +0900 Subject: [PATCH 0354/1172] configure.in: fix message --- configure.in | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/configure.in b/configure.in index 6bf1b48..00211cc 100644 --- a/configure.in +++ b/configure.in @@ -35,16 +35,18 @@ AC_CHECK_HEADERS(tr1/unordered_set) AC_LANG_POP([C++]) AC_CACHE_CHECK([for __sync_* atomic operations], msgpack_cv_atomic_ops, [ - AC_TRY_LINK([ - int atomic_sub(int i) { return __sync_sub_and_fetch(&i, 1); } - int atomic_add(int i) { return __sync_add_and_fetch(&i, 1); } - ], [], msgpack_cv_atomic_ops="yes") - ]) + AC_TRY_LINK([ + int atomic_sub(int i) { return __sync_sub_and_fetch(&i, 1); } + int atomic_add(int i) { return __sync_add_and_fetch(&i, 1); } + ], [], msgpack_cv_atomic_ops="yes") + ]) if test "$msgpack_cv_atomic_ops" != "yes"; then - AC_MSG_ERROR([__sync_* atomic operations are not supported. + AC_MSG_ERROR([__sync_* atomic operations are not supported. + Note that gcc < 4.1 is not supported. -If you are using gcc-4.1 and the CPU architecture is x86, try to add -CFLAGS"--march=i686" and CXXFLAGS="-march=i668" options to ./configure as follows: + +If you are using gcc >= 4.1 and the default target CPU architecture is "i386", try to +add CFLAGS="--march=i686" and CXXFLAGS="-march=i668" options to ./configure as follows: $ ./configure CFLAGS="-march=i686" CXXFLAGS="-march=i686" ]) From 404a3933154dc57b2e650cdabb51f80e4aeb1dbe Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Thu, 21 Jan 2010 21:55:06 +0900 Subject: [PATCH 0355/1172] c, cpp: fix test problem - using LDADD instead of LDFLAGS - fix include path in test.cpp --- c/Makefile.am | 2 +- cpp/Makefile.am | 2 +- cpp/test.cpp | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/c/Makefile.am b/c/Makefile.am index 23995a5..f84759a 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -23,6 +23,6 @@ check_PROGRAMS = \ msgpackc_test_SOURCES = test.cpp msgpackc_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -msgpackc_test_LDFLAGS = libmsgpackc.la -lgtest_main +msgpackc_test_LDADD = libmsgpackc.la -lgtest_main TESTS = $(check_PROGRAMS) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 99f4a32..d4de161 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -39,6 +39,6 @@ check_PROGRAMS = \ msgpack_test_SOURCES = test.cpp msgpack_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -I$(top_srcdir)/cpp -msgpack_test_LDFLAGS = libmsgpack.la -lgtest_main +msgpack_test_LDADD = libmsgpack.la -lgtest_main TESTS = $(check_PROGRAMS) diff --git a/cpp/test.cpp b/cpp/test.cpp index b152e6d..113914a 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -438,7 +438,7 @@ TEST(MSGPACK_STL, simple_buffer_multiset) #ifdef HAVE_TR1_UNORDERED_MAP #include -#include "cpp/type/tr1/unordered_map.hpp" +#include "cpp/msgpack/type/tr1/unordered_map.hpp" TEST(MSGPACK_TR1, simple_buffer_unordered_map) { for (unsigned int k = 0; k < kLoop; k++) { @@ -499,7 +499,7 @@ TEST(MSGPACK_TR1, simple_buffer_unordered_multimap) #ifdef HAVE_TR1_UNORDERED_SET #include -#include "cpp/type/tr1/unordered_set.hpp" +#include "cpp/msgpack/type/tr1/unordered_set.hpp" TEST(MSGPACK_TR1, simple_buffer_unordered_set) { for (unsigned int k = 0; k < kLoop; k++) { From 9e2c9d7812565bfff8004990aef5e3f99f36c2b7 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 25 Jan 2010 12:21:46 +0900 Subject: [PATCH 0356/1172] Tiny fix in README --- python/README | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/README b/python/README index 2b2ebe9..a7e8c53 100644 --- a/python/README +++ b/python/README @@ -2,7 +2,7 @@ MessagePack Python Binding =========================== -:author: Naoki INADA +:author: INADA Naoki :version: 0.1.0 :date: 2009-07-12 @@ -32,7 +32,11 @@ MinGW GCC. TEST ---- -MessagePack uses nosetest for testing. +MessagePack uses `nosetest` for testing. Run test with following command: $ nosetests test + + +.. + vim: filetype=rst From 6f8e7f2709b811068988055c956790cb0537b41b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 25 Jan 2010 12:21:46 +0900 Subject: [PATCH 0357/1172] Tiny fix in README --- README | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README b/README index 2b2ebe9..a7e8c53 100644 --- a/README +++ b/README @@ -2,7 +2,7 @@ MessagePack Python Binding =========================== -:author: Naoki INADA +:author: INADA Naoki :version: 0.1.0 :date: 2009-07-12 @@ -32,7 +32,11 @@ MinGW GCC. TEST ---- -MessagePack uses nosetest for testing. +MessagePack uses `nosetest` for testing. Run test with following command: $ nosetests test + + +.. + vim: filetype=rst From 1a11608f1f5ed4f9ae6dd302b31327145529bf38 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 25 Jan 2010 20:51:55 +0900 Subject: [PATCH 0358/1172] cythoning *.pyx when sdist. --- python/setup_dev.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/python/setup_dev.py b/python/setup_dev.py index 4efc769..66cf27e 100755 --- a/python/setup_dev.py +++ b/python/setup_dev.py @@ -1,15 +1,41 @@ #!/usr/bin/env python # coding: utf-8 -from distutils.core import setup, Extension -from Cython.Distutils import build_ext import os +from glob import glob +from distutils.core import setup, Extension +from distutils.command.sdist import sdist + +try: + from Cython.Distutils import build_ext + import Cython.Compiler.Main as cython_compiler + have_cython = True +except ImportError: + from distutils.command.build_ext import build_ext + have_cython = False version = '0.2.0dev' +# take care of extension modules. +if have_cython: + sources = ['msgpack/_msgpack.pyx'] + + class Sdist(sdist): + def __init__(self, *args, **kwargs): + for src in glob('msgpack/*.pyx'): + cython_compiler.compile(glob('msgpack/*.pyx'), + cython_compiler.default_options) + sdist.__init__(self, *args, **kwargs) +else: + sources = ['msgpack/_msgpack.c'] + + Sdist = sdist + msgpack_mod = Extension('msgpack._msgpack', - sources=['msgpack/_msgpack.pyx'] + sources=sources, ) +del sources + desc = 'MessagePack (de)serializer.' long_desc = desc + """ @@ -25,14 +51,15 @@ What's MessagePack? (from http://msgpack.sourceforge.jp/) """ setup(name='msgpack', - author='Naoki INADA', + author='INADA Naoki', author_email='songofacandy@gmail.com', version=version, - cmdclass={'build_ext': build_ext}, + cmdclass={'build_ext': build_ext, 'sdist': Sdist}, ext_modules=[msgpack_mod], packages=['msgpack'], description=desc, long_description=long_desc, + url="http://msgpack.sourceforge.jp/", classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', From b6a4568b0fcb4433c4e1d033015c881a24bf32a3 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 25 Jan 2010 20:51:55 +0900 Subject: [PATCH 0359/1172] cythoning *.pyx when sdist. --- setup_dev.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/setup_dev.py b/setup_dev.py index 4efc769..66cf27e 100755 --- a/setup_dev.py +++ b/setup_dev.py @@ -1,15 +1,41 @@ #!/usr/bin/env python # coding: utf-8 -from distutils.core import setup, Extension -from Cython.Distutils import build_ext import os +from glob import glob +from distutils.core import setup, Extension +from distutils.command.sdist import sdist + +try: + from Cython.Distutils import build_ext + import Cython.Compiler.Main as cython_compiler + have_cython = True +except ImportError: + from distutils.command.build_ext import build_ext + have_cython = False version = '0.2.0dev' +# take care of extension modules. +if have_cython: + sources = ['msgpack/_msgpack.pyx'] + + class Sdist(sdist): + def __init__(self, *args, **kwargs): + for src in glob('msgpack/*.pyx'): + cython_compiler.compile(glob('msgpack/*.pyx'), + cython_compiler.default_options) + sdist.__init__(self, *args, **kwargs) +else: + sources = ['msgpack/_msgpack.c'] + + Sdist = sdist + msgpack_mod = Extension('msgpack._msgpack', - sources=['msgpack/_msgpack.pyx'] + sources=sources, ) +del sources + desc = 'MessagePack (de)serializer.' long_desc = desc + """ @@ -25,14 +51,15 @@ What's MessagePack? (from http://msgpack.sourceforge.jp/) """ setup(name='msgpack', - author='Naoki INADA', + author='INADA Naoki', author_email='songofacandy@gmail.com', version=version, - cmdclass={'build_ext': build_ext}, + cmdclass={'build_ext': build_ext, 'sdist': Sdist}, ext_modules=[msgpack_mod], packages=['msgpack'], description=desc, long_description=long_desc, + url="http://msgpack.sourceforge.jp/", classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', From e02d20dd747ce70e5e0de71bce244bd86666c6f0 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 25 Jan 2010 20:52:48 +0900 Subject: [PATCH 0360/1172] replace setup by setup_dev. --- python/setup.py | 38 +++++++++++++++++++++---- python/setup_dev.py | 68 --------------------------------------------- 2 files changed, 32 insertions(+), 74 deletions(-) delete mode 100755 python/setup_dev.py diff --git a/python/setup.py b/python/setup.py index c48be14..66cf27e 100755 --- a/python/setup.py +++ b/python/setup.py @@ -1,16 +1,41 @@ #!/usr/bin/env python # coding: utf-8 -from distutils.core import setup, Extension -#from Cython.Distutils import build_ext import os +from glob import glob +from distutils.core import setup, Extension +from distutils.command.sdist import sdist + +try: + from Cython.Distutils import build_ext + import Cython.Compiler.Main as cython_compiler + have_cython = True +except ImportError: + from distutils.command.build_ext import build_ext + have_cython = False version = '0.2.0dev' +# take care of extension modules. +if have_cython: + sources = ['msgpack/_msgpack.pyx'] + + class Sdist(sdist): + def __init__(self, *args, **kwargs): + for src in glob('msgpack/*.pyx'): + cython_compiler.compile(glob('msgpack/*.pyx'), + cython_compiler.default_options) + sdist.__init__(self, *args, **kwargs) +else: + sources = ['msgpack/_msgpack.c'] + + Sdist = sdist + msgpack_mod = Extension('msgpack._msgpack', - #sources=['msgpack/_msgpack.pyx'] - sources=['msgpack/_msgpack.c'] + sources=sources, ) +del sources + desc = 'MessagePack (de)serializer.' long_desc = desc + """ @@ -26,14 +51,15 @@ What's MessagePack? (from http://msgpack.sourceforge.jp/) """ setup(name='msgpack', - author='Naoki INADA', + author='INADA Naoki', author_email='songofacandy@gmail.com', version=version, - #cmdclass={'build_ext': build_ext}, + cmdclass={'build_ext': build_ext, 'sdist': Sdist}, ext_modules=[msgpack_mod], packages=['msgpack'], description=desc, long_description=long_desc, + url="http://msgpack.sourceforge.jp/", classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', diff --git a/python/setup_dev.py b/python/setup_dev.py deleted file mode 100755 index 66cf27e..0000000 --- a/python/setup_dev.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -import os -from glob import glob -from distutils.core import setup, Extension -from distutils.command.sdist import sdist - -try: - from Cython.Distutils import build_ext - import Cython.Compiler.Main as cython_compiler - have_cython = True -except ImportError: - from distutils.command.build_ext import build_ext - have_cython = False - -version = '0.2.0dev' - -# take care of extension modules. -if have_cython: - sources = ['msgpack/_msgpack.pyx'] - - class Sdist(sdist): - def __init__(self, *args, **kwargs): - for src in glob('msgpack/*.pyx'): - cython_compiler.compile(glob('msgpack/*.pyx'), - cython_compiler.default_options) - sdist.__init__(self, *args, **kwargs) -else: - sources = ['msgpack/_msgpack.c'] - - Sdist = sdist - -msgpack_mod = Extension('msgpack._msgpack', - sources=sources, - ) -del sources - - -desc = 'MessagePack (de)serializer.' -long_desc = desc + """ - -MessagePack_ (de)serializer for Python. - -.. _MessagePack: http://msgpack.sourceforge.jp/ - -What's MessagePack? (from http://msgpack.sourceforge.jp/) - - MessagePack is a binary-based efficient data interchange format that is - focused on high performance. It is like JSON, but very fast and small. -""" - -setup(name='msgpack', - author='INADA Naoki', - author_email='songofacandy@gmail.com', - version=version, - cmdclass={'build_ext': build_ext, 'sdist': Sdist}, - ext_modules=[msgpack_mod], - packages=['msgpack'], - description=desc, - long_description=long_desc, - url="http://msgpack.sourceforge.jp/", - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: Apache Software License', - ] - ) From de508dcc1047c7bf065fc5476a48f94f725b059a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 25 Jan 2010 20:52:48 +0900 Subject: [PATCH 0361/1172] replace setup by setup_dev. --- setup.py | 38 ++++++++++++++++++++++++----- setup_dev.py | 68 ---------------------------------------------------- 2 files changed, 32 insertions(+), 74 deletions(-) delete mode 100755 setup_dev.py diff --git a/setup.py b/setup.py index c48be14..66cf27e 100755 --- a/setup.py +++ b/setup.py @@ -1,16 +1,41 @@ #!/usr/bin/env python # coding: utf-8 -from distutils.core import setup, Extension -#from Cython.Distutils import build_ext import os +from glob import glob +from distutils.core import setup, Extension +from distutils.command.sdist import sdist + +try: + from Cython.Distutils import build_ext + import Cython.Compiler.Main as cython_compiler + have_cython = True +except ImportError: + from distutils.command.build_ext import build_ext + have_cython = False version = '0.2.0dev' +# take care of extension modules. +if have_cython: + sources = ['msgpack/_msgpack.pyx'] + + class Sdist(sdist): + def __init__(self, *args, **kwargs): + for src in glob('msgpack/*.pyx'): + cython_compiler.compile(glob('msgpack/*.pyx'), + cython_compiler.default_options) + sdist.__init__(self, *args, **kwargs) +else: + sources = ['msgpack/_msgpack.c'] + + Sdist = sdist + msgpack_mod = Extension('msgpack._msgpack', - #sources=['msgpack/_msgpack.pyx'] - sources=['msgpack/_msgpack.c'] + sources=sources, ) +del sources + desc = 'MessagePack (de)serializer.' long_desc = desc + """ @@ -26,14 +51,15 @@ What's MessagePack? (from http://msgpack.sourceforge.jp/) """ setup(name='msgpack', - author='Naoki INADA', + author='INADA Naoki', author_email='songofacandy@gmail.com', version=version, - #cmdclass={'build_ext': build_ext}, + cmdclass={'build_ext': build_ext, 'sdist': Sdist}, ext_modules=[msgpack_mod], packages=['msgpack'], description=desc, long_description=long_desc, + url="http://msgpack.sourceforge.jp/", classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', diff --git a/setup_dev.py b/setup_dev.py deleted file mode 100755 index 66cf27e..0000000 --- a/setup_dev.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -import os -from glob import glob -from distutils.core import setup, Extension -from distutils.command.sdist import sdist - -try: - from Cython.Distutils import build_ext - import Cython.Compiler.Main as cython_compiler - have_cython = True -except ImportError: - from distutils.command.build_ext import build_ext - have_cython = False - -version = '0.2.0dev' - -# take care of extension modules. -if have_cython: - sources = ['msgpack/_msgpack.pyx'] - - class Sdist(sdist): - def __init__(self, *args, **kwargs): - for src in glob('msgpack/*.pyx'): - cython_compiler.compile(glob('msgpack/*.pyx'), - cython_compiler.default_options) - sdist.__init__(self, *args, **kwargs) -else: - sources = ['msgpack/_msgpack.c'] - - Sdist = sdist - -msgpack_mod = Extension('msgpack._msgpack', - sources=sources, - ) -del sources - - -desc = 'MessagePack (de)serializer.' -long_desc = desc + """ - -MessagePack_ (de)serializer for Python. - -.. _MessagePack: http://msgpack.sourceforge.jp/ - -What's MessagePack? (from http://msgpack.sourceforge.jp/) - - MessagePack is a binary-based efficient data interchange format that is - focused on high performance. It is like JSON, but very fast and small. -""" - -setup(name='msgpack', - author='INADA Naoki', - author_email='songofacandy@gmail.com', - version=version, - cmdclass={'build_ext': build_ext, 'sdist': Sdist}, - ext_modules=[msgpack_mod], - packages=['msgpack'], - description=desc, - long_description=long_desc, - url="http://msgpack.sourceforge.jp/", - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: Apache Software License', - ] - ) From 34b3bbc8839f63e3e624dc0ce42525657cdcdfa7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 29 Jan 2010 17:23:48 +0900 Subject: [PATCH 0362/1172] ruby: add Unpacker#each --- ruby/unpack.c | 127 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 120 insertions(+), 7 deletions(-) diff --git a/ruby/unpack.c b/ruby/unpack.c index e9d6494..0f1af38 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -18,10 +18,16 @@ #include "ruby.h" #include "msgpack/unpack_define.h" +static ID s_sysread; typedef struct { int finished; VALUE source; + size_t offset; + size_t parsed; + VALUE buffer; + VALUE stream; + VALUE streambuf; } unpack_user; @@ -144,6 +150,9 @@ static void MessagePack_Unpacker_free(void* data) static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp) { unsigned int i; + rb_gc_mark(mp->user.buffer); + rb_gc_mark(mp->user.stream); + rb_gc_mark(mp->user.streambuf); for(i=0; i < mp->top; ++i) { rb_gc_mark(mp->stack[i].obj); rb_gc_mark(mp->stack[i].map_key); /* maybe map_key is not initialized */ @@ -164,14 +173,32 @@ static VALUE MessagePack_Unpacker_reset(VALUE self) UNPACKER(self, mp); template_init(mp); init_stack(mp); - unpack_user u = {0, Qnil}; - mp->user = u; + mp->user.finished = 0; return self; } -static VALUE MessagePack_Unpacker_initialize(VALUE self) +static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) { - return MessagePack_Unpacker_reset(self); + VALUE stream; + switch(argc) { + case 0: + stream = Qnil; + break; + case 1: + stream = argv[0]; + break; + default: + rb_raise(rb_eArgError, "wrong number of arguments (%d for 0)", argc); + } + + MessagePack_Unpacker_reset(self); + UNPACKER(self, mp); + mp->user.offset = 0; + mp->user.parsed = 0; + mp->user.buffer = rb_str_new("",0); + mp->user.stream = stream; + mp->user.streambuf = rb_str_new("",0); + return self; } @@ -249,6 +276,87 @@ static VALUE MessagePack_Unpacker_data(VALUE self) } +static VALUE MessagePack_Unpacker_feed(VALUE self, VALUE data) +{ + UNPACKER(self, mp); + StringValue(data); + rb_str_cat(mp->user.buffer, RSTRING_PTR(data), RSTRING_LEN(data)); + return Qnil; +} + +static VALUE MessagePack_Unpacker_stream_get(VALUE self) +{ + UNPACKER(self, mp); + return mp->user.stream; +} + +static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val) +{ + UNPACKER(self, mp); + return mp->user.stream = val; +} + +static VALUE MessagePack_Unpacker_fill(VALUE self) +{ + UNPACKER(self, mp); + + if(mp->user.stream == Qnil) { + return Qnil; + } + + size_t len; + if(RSTRING_LEN(mp->user.buffer) == 0) { + rb_funcall(mp->user.stream, s_sysread, 2, LONG2FIX(64*1024), mp->user.buffer); + len = RSTRING_LEN(mp->user.buffer); + } else { + rb_funcall(mp->user.stream, s_sysread, 2, LONG2FIX(64*1024), mp->user.streambuf); + len = RSTRING_LEN(mp->user.streambuf); + rb_str_cat(mp->user.buffer, RSTRING_PTR(mp->user.streambuf), RSTRING_LEN(mp->user.streambuf)); + } + + return LONG2FIX(len); +} + +static VALUE MessagePack_Unpacker_each(VALUE self) +{ + UNPACKER(self, mp); + int ret; + +#ifdef RETURN_ENUMERATOR + RETURN_ENUMERATOR(self, 0, 0); +#endif + + while(1) { + if(RSTRING_LEN(mp->user.buffer) <= mp->user.offset) { + do_fill: + { + VALUE len = MessagePack_Unpacker_fill(self); + if(len == Qnil || FIX2LONG(len) == 0) { + break; + } + } + } + + mp->user.source = mp->user.buffer; + ret = template_execute(mp, RSTRING_PTR(mp->user.buffer), RSTRING_LEN(mp->user.buffer), &mp->user.offset); + mp->user.source = Qnil; + + if(ret < 0) { + rb_raise(eUnpackError, "parse error."); + } else if(ret > 0) { + VALUE data = template_data(mp); + template_init(mp); + init_stack(mp); + rb_yield(data); + } else { + goto do_fill; + } + } + + return Qnil; +} + + static VALUE MessagePack_unpack_impl(VALUE args) { msgpack_unpack_t* mp = (msgpack_unpack_t*)((VALUE*)args)[0]; @@ -292,7 +400,7 @@ static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) msgpack_unpack_t mp; template_init(&mp); init_stack(&mp); - unpack_user u = {0, Qnil}; + unpack_user u = {0, Qnil, 0, 0, Qnil, Qnil, Qnil}; mp.user = u; rb_gc_disable(); @@ -313,17 +421,22 @@ static VALUE MessagePack_unpack(VALUE self, VALUE data) void Init_msgpack_unpack(VALUE mMessagePack) { + s_sysread = rb_intern("sysread"); eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); - rb_define_method(cUnpacker, "initialize", MessagePack_Unpacker_initialize, 0); + rb_define_method(cUnpacker, "initialize", MessagePack_Unpacker_initialize, -1); rb_define_method(cUnpacker, "execute", MessagePack_Unpacker_execute, 2); rb_define_method(cUnpacker, "execute_limit", MessagePack_Unpacker_execute_limit, 3); rb_define_method(cUnpacker, "finished?", MessagePack_Unpacker_finished_p, 0); rb_define_method(cUnpacker, "data", MessagePack_Unpacker_data, 0); rb_define_method(cUnpacker, "reset", MessagePack_Unpacker_reset, 0); + rb_define_method(cUnpacker, "feed", MessagePack_Unpacker_feed, 1); + rb_define_method(cUnpacker, "fill", MessagePack_Unpacker_fill, 0); + rb_define_method(cUnpacker, "each", MessagePack_Unpacker_each, 0); + rb_define_method(cUnpacker, "stream", MessagePack_Unpacker_stream_get, 0); + rb_define_method(cUnpacker, "stream=", MessagePack_Unpacker_stream_set, 1); rb_define_module_function(mMessagePack, "unpack", MessagePack_unpack, 1); rb_define_module_function(mMessagePack, "unpack_limit", MessagePack_unpack_limit, 2); } - From 7df60b259b55b6294c19315350ec312809107117 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 2 Feb 2010 10:52:42 +0900 Subject: [PATCH 0363/1172] MSVC2005 compatibility (@hotpepsi++) --- c/object.h | 1 - c/zone.c | 4 ---- c/zone.h | 1 + cpp/type/vector.hpp | 14 +++++++++----- msgpack/sysdep.h | 11 +++++++++++ 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/c/object.h b/c/object.h index 27b593e..9a014be 100644 --- a/c/object.h +++ b/c/object.h @@ -19,7 +19,6 @@ #define MSGPACK_OBJECT_H__ #include "msgpack/zone.h" -#include "msgpack/sysdep.h" #include #ifdef __cplusplus diff --git a/c/zone.c b/c/zone.c index 2d8bc58..3d0634e 100644 --- a/c/zone.c +++ b/c/zone.c @@ -104,7 +104,6 @@ static inline void init_finalizer_array(msgpack_zone_finalizer_array* fa) static inline void call_finalizer_array(msgpack_zone_finalizer_array* fa) { - // 逆順ã«å‘¼ã³å‡ºã— msgpack_zone_finalizer* fin = fa->tail; for(; fin != fa->array; --fin) { (*(fin-1)->func)((fin-1)->data); @@ -132,9 +131,6 @@ bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, size_t nnext; if(nused == 0) { - // åˆå›žã®å‘¼ã³å‡ºã—：fa->tail == fa->end == fa->array == NULL - - // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ nnext = (sizeof(msgpack_zone_finalizer) < 72/2) ? 72 / sizeof(msgpack_zone_finalizer) : 8; diff --git a/c/zone.h b/c/zone.h index e07e762..475ae51 100644 --- a/c/zone.h +++ b/c/zone.h @@ -18,6 +18,7 @@ #ifndef MSGPACK_ZONE_H__ #define MSGPACK_ZONE_H__ +#include "msgpack/sysdep.h" #include #include diff --git a/cpp/type/vector.hpp b/cpp/type/vector.hpp index 754cdc0..385d070 100644 --- a/cpp/type/vector.hpp +++ b/cpp/type/vector.hpp @@ -29,11 +29,15 @@ inline std::vector& operator>> (object o, std::vector& v) { if(o.type != type::ARRAY) { throw type_error(); } v.resize(o.via.array.size); - object* p = o.via.array.ptr; - object* const pend = o.via.array.ptr + o.via.array.size; - T* it = &v.front(); - for(; p < pend; ++p, ++it) { - p->convert(it); + if(o.via.array.size > 0) { + object* p = o.via.array.ptr; + object* const pend = o.via.array.ptr + o.via.array.size; + T* it = &v[0]; + do { + p->convert(it); + ++p; + ++it; + } while(p < pend); } return v; } diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h index 106158e..e9540d5 100644 --- a/msgpack/sysdep.h +++ b/msgpack/sysdep.h @@ -46,6 +46,17 @@ typedef unsigned int _msgpack_atomic_counter_t; #endif +#ifdef __cplusplus +/* numeric_limits::min,max */ +#ifdef max +#undef max +#endif +#ifdef min +#undef min +#endif +#endif + + #ifdef _WIN32 #include #else From 66ef6c9e4cf66626ec5af498096b35abb8a050c0 Mon Sep 17 00:00:00 2001 From: firewood Date: Fri, 5 Feb 2010 22:49:00 +0900 Subject: [PATCH 0364/1172] undef after including winsock2.h. --- msgpack/sysdep.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h index e9540d5..c69c69a 100644 --- a/msgpack/sysdep.h +++ b/msgpack/sysdep.h @@ -46,6 +46,9 @@ typedef unsigned int _msgpack_atomic_counter_t; #endif +#ifdef _WIN32 +#include + #ifdef __cplusplus /* numeric_limits::min,max */ #ifdef max @@ -56,9 +59,6 @@ typedef unsigned int _msgpack_atomic_counter_t; #endif #endif - -#ifdef _WIN32 -#include #else #include /* __BYTE_ORDER */ #endif From 9a034234d8955b2f6f3877d9cb39993ffd5981a9 Mon Sep 17 00:00:00 2001 From: firewood Date: Fri, 5 Feb 2010 22:49:28 +0900 Subject: [PATCH 0365/1172] add vc2005 project file. --- msgpack_vc8.vcproj | 275 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 msgpack_vc8.vcproj diff --git a/msgpack_vc8.vcproj b/msgpack_vc8.vcproj new file mode 100644 index 0000000..5daec01 --- /dev/null +++ b/msgpack_vc8.vcproj @@ -0,0 +1,275 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From ca0b085d156260b6d916139495a6d306a1e7f886 Mon Sep 17 00:00:00 2001 From: firewood Date: Fri, 5 Feb 2010 22:50:05 +0900 Subject: [PATCH 0366/1172] add a blanc after Japanese comment --- c/vrefbuffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c/vrefbuffer.c b/c/vrefbuffer.c index b970ece..b528874 100644 --- a/c/vrefbuffer.c +++ b/c/vrefbuffer.c @@ -30,7 +30,7 @@ bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, vbuf->chunk_size = chunk_size; vbuf->ref_size = ref_size; - // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ size_t nfirst = (sizeof(struct iovec) < 72/2) ? 72 / sizeof(struct iovec) : 8; From cd10fbc1fe39308045f0b7bf9cce5f8153f25f1d Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 6 Feb 2010 20:09:41 +0900 Subject: [PATCH 0367/1172] removed symbolic links --- c/msgpack | 1 - c/{ => msgpack}/object.h | 0 c/{ => msgpack}/pack.h | 0 c/{ => msgpack}/sbuffer.h | 0 c/{ => msgpack}/unpack.h | 0 c/{ => msgpack}/vrefbuffer.h | 0 c/{ => msgpack}/zone.h | 0 cpp/msgpack | 1 - cpp/{ => msgpack}/object.hpp | 0 cpp/{ => msgpack}/pack.hpp | 0 cpp/{ => msgpack}/sbuffer.hpp | 0 cpp/{ => msgpack}/type.hpp | 0 cpp/{ => msgpack}/type/bool.hpp | 0 cpp/{ => msgpack}/type/define.hpp.erb | 0 cpp/{ => msgpack}/type/deque.hpp | 0 cpp/{ => msgpack}/type/float.hpp | 0 cpp/{ => msgpack}/type/int.hpp | 0 cpp/{ => msgpack}/type/list.hpp | 0 cpp/{ => msgpack}/type/map.hpp | 0 cpp/{ => msgpack}/type/nil.hpp | 0 cpp/{ => msgpack}/type/pair.hpp | 0 cpp/{ => msgpack}/type/raw.hpp | 0 cpp/{ => msgpack}/type/set.hpp | 0 cpp/{ => msgpack}/type/string.hpp | 0 cpp/{ => msgpack}/type/tr1/unordered_map.hpp | 0 cpp/{ => msgpack}/type/tr1/unordered_set.hpp | 0 cpp/{ => msgpack}/type/tuple.hpp.erb | 0 cpp/{ => msgpack}/type/vector.hpp | 0 cpp/{ => msgpack}/unpack.hpp | 0 cpp/{ => msgpack}/vrefbuffer.hpp | 0 cpp/{ => msgpack}/zone.hpp.erb | 0 31 files changed, 2 deletions(-) delete mode 120000 c/msgpack rename c/{ => msgpack}/object.h (100%) rename c/{ => msgpack}/pack.h (100%) rename c/{ => msgpack}/sbuffer.h (100%) rename c/{ => msgpack}/unpack.h (100%) rename c/{ => msgpack}/vrefbuffer.h (100%) rename c/{ => msgpack}/zone.h (100%) delete mode 120000 cpp/msgpack rename cpp/{ => msgpack}/object.hpp (100%) rename cpp/{ => msgpack}/pack.hpp (100%) rename cpp/{ => msgpack}/sbuffer.hpp (100%) rename cpp/{ => msgpack}/type.hpp (100%) rename cpp/{ => msgpack}/type/bool.hpp (100%) rename cpp/{ => msgpack}/type/define.hpp.erb (100%) rename cpp/{ => msgpack}/type/deque.hpp (100%) rename cpp/{ => msgpack}/type/float.hpp (100%) rename cpp/{ => msgpack}/type/int.hpp (100%) rename cpp/{ => msgpack}/type/list.hpp (100%) rename cpp/{ => msgpack}/type/map.hpp (100%) rename cpp/{ => msgpack}/type/nil.hpp (100%) rename cpp/{ => msgpack}/type/pair.hpp (100%) rename cpp/{ => msgpack}/type/raw.hpp (100%) rename cpp/{ => msgpack}/type/set.hpp (100%) rename cpp/{ => msgpack}/type/string.hpp (100%) rename cpp/{ => msgpack}/type/tr1/unordered_map.hpp (100%) rename cpp/{ => msgpack}/type/tr1/unordered_set.hpp (100%) rename cpp/{ => msgpack}/type/tuple.hpp.erb (100%) rename cpp/{ => msgpack}/type/vector.hpp (100%) rename cpp/{ => msgpack}/unpack.hpp (100%) rename cpp/{ => msgpack}/vrefbuffer.hpp (100%) rename cpp/{ => msgpack}/zone.hpp.erb (100%) diff --git a/c/msgpack b/c/msgpack deleted file mode 120000 index 945c9b4..0000000 --- a/c/msgpack +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/c/object.h b/c/msgpack/object.h similarity index 100% rename from c/object.h rename to c/msgpack/object.h diff --git a/c/pack.h b/c/msgpack/pack.h similarity index 100% rename from c/pack.h rename to c/msgpack/pack.h diff --git a/c/sbuffer.h b/c/msgpack/sbuffer.h similarity index 100% rename from c/sbuffer.h rename to c/msgpack/sbuffer.h diff --git a/c/unpack.h b/c/msgpack/unpack.h similarity index 100% rename from c/unpack.h rename to c/msgpack/unpack.h diff --git a/c/vrefbuffer.h b/c/msgpack/vrefbuffer.h similarity index 100% rename from c/vrefbuffer.h rename to c/msgpack/vrefbuffer.h diff --git a/c/zone.h b/c/msgpack/zone.h similarity index 100% rename from c/zone.h rename to c/msgpack/zone.h diff --git a/cpp/msgpack b/cpp/msgpack deleted file mode 120000 index 945c9b4..0000000 --- a/cpp/msgpack +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/cpp/object.hpp b/cpp/msgpack/object.hpp similarity index 100% rename from cpp/object.hpp rename to cpp/msgpack/object.hpp diff --git a/cpp/pack.hpp b/cpp/msgpack/pack.hpp similarity index 100% rename from cpp/pack.hpp rename to cpp/msgpack/pack.hpp diff --git a/cpp/sbuffer.hpp b/cpp/msgpack/sbuffer.hpp similarity index 100% rename from cpp/sbuffer.hpp rename to cpp/msgpack/sbuffer.hpp diff --git a/cpp/type.hpp b/cpp/msgpack/type.hpp similarity index 100% rename from cpp/type.hpp rename to cpp/msgpack/type.hpp diff --git a/cpp/type/bool.hpp b/cpp/msgpack/type/bool.hpp similarity index 100% rename from cpp/type/bool.hpp rename to cpp/msgpack/type/bool.hpp diff --git a/cpp/type/define.hpp.erb b/cpp/msgpack/type/define.hpp.erb similarity index 100% rename from cpp/type/define.hpp.erb rename to cpp/msgpack/type/define.hpp.erb diff --git a/cpp/type/deque.hpp b/cpp/msgpack/type/deque.hpp similarity index 100% rename from cpp/type/deque.hpp rename to cpp/msgpack/type/deque.hpp diff --git a/cpp/type/float.hpp b/cpp/msgpack/type/float.hpp similarity index 100% rename from cpp/type/float.hpp rename to cpp/msgpack/type/float.hpp diff --git a/cpp/type/int.hpp b/cpp/msgpack/type/int.hpp similarity index 100% rename from cpp/type/int.hpp rename to cpp/msgpack/type/int.hpp diff --git a/cpp/type/list.hpp b/cpp/msgpack/type/list.hpp similarity index 100% rename from cpp/type/list.hpp rename to cpp/msgpack/type/list.hpp diff --git a/cpp/type/map.hpp b/cpp/msgpack/type/map.hpp similarity index 100% rename from cpp/type/map.hpp rename to cpp/msgpack/type/map.hpp diff --git a/cpp/type/nil.hpp b/cpp/msgpack/type/nil.hpp similarity index 100% rename from cpp/type/nil.hpp rename to cpp/msgpack/type/nil.hpp diff --git a/cpp/type/pair.hpp b/cpp/msgpack/type/pair.hpp similarity index 100% rename from cpp/type/pair.hpp rename to cpp/msgpack/type/pair.hpp diff --git a/cpp/type/raw.hpp b/cpp/msgpack/type/raw.hpp similarity index 100% rename from cpp/type/raw.hpp rename to cpp/msgpack/type/raw.hpp diff --git a/cpp/type/set.hpp b/cpp/msgpack/type/set.hpp similarity index 100% rename from cpp/type/set.hpp rename to cpp/msgpack/type/set.hpp diff --git a/cpp/type/string.hpp b/cpp/msgpack/type/string.hpp similarity index 100% rename from cpp/type/string.hpp rename to cpp/msgpack/type/string.hpp diff --git a/cpp/type/tr1/unordered_map.hpp b/cpp/msgpack/type/tr1/unordered_map.hpp similarity index 100% rename from cpp/type/tr1/unordered_map.hpp rename to cpp/msgpack/type/tr1/unordered_map.hpp diff --git a/cpp/type/tr1/unordered_set.hpp b/cpp/msgpack/type/tr1/unordered_set.hpp similarity index 100% rename from cpp/type/tr1/unordered_set.hpp rename to cpp/msgpack/type/tr1/unordered_set.hpp diff --git a/cpp/type/tuple.hpp.erb b/cpp/msgpack/type/tuple.hpp.erb similarity index 100% rename from cpp/type/tuple.hpp.erb rename to cpp/msgpack/type/tuple.hpp.erb diff --git a/cpp/type/vector.hpp b/cpp/msgpack/type/vector.hpp similarity index 100% rename from cpp/type/vector.hpp rename to cpp/msgpack/type/vector.hpp diff --git a/cpp/unpack.hpp b/cpp/msgpack/unpack.hpp similarity index 100% rename from cpp/unpack.hpp rename to cpp/msgpack/unpack.hpp diff --git a/cpp/vrefbuffer.hpp b/cpp/msgpack/vrefbuffer.hpp similarity index 100% rename from cpp/vrefbuffer.hpp rename to cpp/msgpack/vrefbuffer.hpp diff --git a/cpp/zone.hpp.erb b/cpp/msgpack/zone.hpp.erb similarity index 100% rename from cpp/zone.hpp.erb rename to cpp/msgpack/zone.hpp.erb From d5609f3207ee8b98dc1e2032cd31a23380ad5402 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 6 Feb 2010 21:28:27 +0900 Subject: [PATCH 0368/1172] ruby: makegem.sh --- ruby/{gem => }/README | 0 ruby/gem/Rakefile | 133 ------------------------- ruby/gengem.sh | 21 ---- ruby/makegem.sh | 21 ++++ ruby/msgpack.gemspec | 26 +++-- ruby/{test_case.rb => msgpack_test.rb} | 0 ruby/{gem => }/test/test_helper.rb | 0 ruby/test_format.rb | 122 ----------------------- ruby/test_pack.rb | 58 ----------- 9 files changed, 36 insertions(+), 345 deletions(-) rename ruby/{gem => }/README (100%) delete mode 100644 ruby/gem/Rakefile delete mode 100755 ruby/gengem.sh create mode 100755 ruby/makegem.sh mode change 100755 => 100644 ruby/msgpack.gemspec rename ruby/{test_case.rb => msgpack_test.rb} (100%) rename ruby/{gem => }/test/test_helper.rb (100%) delete mode 100644 ruby/test_format.rb delete mode 100644 ruby/test_pack.rb diff --git a/ruby/gem/README b/ruby/README similarity index 100% rename from ruby/gem/README rename to ruby/README diff --git a/ruby/gem/Rakefile b/ruby/gem/Rakefile deleted file mode 100644 index f2fd684..0000000 --- a/ruby/gem/Rakefile +++ /dev/null @@ -1,133 +0,0 @@ -require 'rubygems' -require 'rake' -require 'rake/clean' -require 'rake/testtask' -require 'rake/packagetask' -require 'rake/gempackagetask' -require 'rake/rdoctask' -require 'rake/contrib/rubyforgepublisher' -require 'rake/contrib/sshpublisher' -require 'fileutils' -include FileUtils - -NAME = "msgpack" -AUTHOR = "FURUHASHI Sadayuki" -EMAIL = "frsyuki _at_ users.sourceforge.jp" -DESCRIPTION = "Binary-based efficient data interchange format." -RUBYFORGE_PROJECT = "msgpack" -HOMEPATH = "http://msgpack.sourceforge.jp/" -BIN_FILES = %w( ) -VERS = "0.3.2" - -#REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil -REV = nil -CLEAN.include ['**/.*.sw?', '*.gem', '.config'] -RDOC_OPTS = [ - '--title', "#{NAME} documentation", - "--charset", "utf-8", - "--opname", "index.html", - "--line-numbers", - "--main", "README", - "--inline-source", -] - -task :default => [:test] -task :package => [:clean] - -Rake::TestTask.new("test") do |t| - t.libs << "test" - t.pattern = "test/**/*_test.rb" - t.verbose = true -end - -spec = Gem::Specification.new do |s| - s.name = NAME - s.version = VERS - s.platform = Gem::Platform::RUBY - s.has_rdoc = false - s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] - s.rdoc_options += RDOC_OPTS + ['--exclude', '^(examples|extras)/'] - s.summary = DESCRIPTION - s.description = DESCRIPTION - s.author = AUTHOR - s.email = EMAIL - s.homepage = HOMEPATH - s.executables = BIN_FILES - s.rubyforge_project = RUBYFORGE_PROJECT - s.bindir = "bin" - s.require_path = "ext" - s.autorequire = "" - s.test_files = Dir["test/test_*.rb"] - - #s.add_dependency('activesupport', '>=1.3.1') - #s.required_ruby_version = '>= 1.8.2' - - s.files = %w(README ChangeLog Rakefile) + - Dir.glob("{bin,doc,test,lib,templates,generator,extras,website,script}/**/*") + - Dir.glob("ext/**/*.{h,c,rb}") + - Dir.glob("examples/**/*.rb") + - Dir.glob("tools/*.rb") + - Dir.glob("msgpack/*.h") - - s.extensions = FileList["ext/**/extconf.rb"].to_a -end - -Rake::GemPackageTask.new(spec) do |p| - p.need_tar = true - p.gem_spec = spec -end - -task :install do - name = "#{NAME}-#{VERS}.gem" - sh %{rake package} - sh %{sudo gem install pkg/#{name}} -end - -task :uninstall => [:clean] do - sh %{sudo gem uninstall #{NAME}} -end - - -#Rake::RDocTask.new do |rdoc| -# rdoc.rdoc_dir = 'html' -# rdoc.options += RDOC_OPTS -# rdoc.template = "resh" -# #rdoc.template = "#{ENV['template']}.rb" if ENV['template'] -# if ENV['DOC_FILES'] -# rdoc.rdoc_files.include(ENV['DOC_FILES'].split(/,\s*/)) -# else -# rdoc.rdoc_files.include('README', 'ChangeLog') -# rdoc.rdoc_files.include('lib/**/*.rb') -# rdoc.rdoc_files.include('ext/**/*.c') -# end -#end - -desc "Publish to RubyForge" -task :rubyforge => [:rdoc, :package] do - require 'rubyforge' - Rake::RubyForgePublisher.new(RUBYFORGE_PROJECT, 'frsyuki').upload -end - -desc 'Package and upload the release to rubyforge.' -task :release => [:clean, :package] do |t| - v = ENV["VERSION"] or abort "Must supply VERSION=x.y.z" - abort "Versions don't match #{v} vs #{VERS}" unless v == VERS - pkg = "pkg/#{NAME}-#{VERS}" - - rf = RubyForge.new - puts "Logging in" - rf.login - - c = rf.userconfig -# c["release_notes"] = description if description -# c["release_changes"] = changes if changes - c["preformatted"] = true - - files = [ - "#{pkg}.tgz", - "#{pkg}.gem" - ].compact - - puts "Releasing #{NAME} v. #{VERS}" - rf.add_release RUBYFORGE_PROJECT, NAME, VERS, *files -end diff --git a/ruby/gengem.sh b/ruby/gengem.sh deleted file mode 100755 index 359debf..0000000 --- a/ruby/gengem.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh - -mkdir -p gem/ext -mkdir -p gem/msgpack -cp extconf.rb gem/ext/ -cp pack.c gem/ext/ -cp pack.h gem/ext/ -cp rbinit.c gem/ext/ -cp unpack.c gem/ext/ -cp unpack.h gem/ext/ -cat test_case.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FILE__) + '\/test_helper.rb'/" > gem/test/msgpack_test.rb -cp ../AUTHORS gem/AUTHORS -cp ../ChangeLog gem/ChangeLog -cp ../msgpack/pack_define.h gem/msgpack/ -cp ../msgpack/pack_template.h gem/msgpack/ -cp ../msgpack/unpack_define.h gem/msgpack/ -cp ../msgpack/unpack_template.h gem/msgpack/ -cp ../msgpack/sysdep.h gem/msgpack/ - -cd gem && rake --trace package - diff --git a/ruby/makegem.sh b/ruby/makegem.sh new file mode 100755 index 0000000..fd0db79 --- /dev/null +++ b/ruby/makegem.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +mkdir -p ext +mkdir -p msgpack +cp extconf.rb ext/ +cp pack.c ext/ +cp pack.h ext/ +cp rbinit.c ext/ +cp unpack.c ext/ +cp unpack.h ext/ +cp ../AUTHORS ./ +cp ../ChangeLog ./ +cp ../msgpack/pack_define.h msgpack/ +cp ../msgpack/pack_template.h msgpack/ +cp ../msgpack/unpack_define.h msgpack/ +cp ../msgpack/unpack_template.h msgpack/ +cp ../msgpack/sysdep.h msgpack/ +cat msgpack_test.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FILE__) + '\/test_helper.rb'/" > test/msgpack_test.rb + +gem build msgpack.gemspec + diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec old mode 100755 new mode 100644 index 59186a4..7273881 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,12 +1,16 @@ -Gem::Specification.new do |s| - s.platform = Gem::Platform::CURRENT - s.name = "msgpack" - s.version = "0.3.2" - s.summary = "MessagePack" - s.author = "FURUHASHI Sadayuki" - s.email = "frsyuki@users.sourceforge.jp" - s.homepage = "http://msgpack.sourceforge.jp/" - s.rubyforge_project = "msgpack" - s.require_paths = ["lib", "ext"] - s.files = ["lib/**/*", "ext/**/*"].map {|g| Dir.glob(g) }.flatten +Gem::Specification.new do |s| + s.platform = Gem::Platform::RUBY + s.name = "msgpack" + s.version = "0.3.3" + s.summary = "MessagePack, a binary-based efficient data interchange format." + s.author = "FURUHASHI Sadayuki" + s.email = "frsyuki@users.sourceforge.jp" + s.homepage = "http://msgpack.sourceforge.jp/" + s.rubyforge_project = "msgpack" + s.has_rdoc = false + s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] + s.require_paths = ["lib", "ext"] + s.files = Dir["lib/**/*", "ext/**/*", "msgpack/**/*", "test/**/*"] + s.test_files = Dir["test/test_*.rb"] + s.extensions = Dir["ext/**/extconf.rb"] end diff --git a/ruby/test_case.rb b/ruby/msgpack_test.rb similarity index 100% rename from ruby/test_case.rb rename to ruby/msgpack_test.rb diff --git a/ruby/gem/test/test_helper.rb b/ruby/test/test_helper.rb similarity index 100% rename from ruby/gem/test/test_helper.rb rename to ruby/test/test_helper.rb diff --git a/ruby/test_format.rb b/ruby/test_format.rb deleted file mode 100644 index 99a27d1..0000000 --- a/ruby/test_format.rb +++ /dev/null @@ -1,122 +0,0 @@ -require 'msgpack' - -@up = MessagePack::Unpacker.new - -def check(bytes, should) - puts "----" - @up.reset - src = bytes.pack('C*') - ret = @up.execute(src, 0) - if ret != src.length - puts "** EXTRA BYTES **" - end - puts bytes.map{|x|"%x"%x}.join(' ') - data = @up.data - p data - if data != should - puts "** TEST FAILED **" - p should - end -end - -# SimpleValue -check([ - 0x93, 0xc0, 0xc2, 0xc3, -], [nil,false,true]) - -# Fixnum -check([ - 0x92, - 0x93, 0x00, 0x40, 0x7f, - 0x93, 0xe0, 0xf0, 0xff, -], [[0,64,127], [-32,-16,-1]]) - -# FixArray -check([ - 0x92, - 0x90, - 0x91, - 0x91, 0xc0, -], [[],[[nil]]]) - - -# FixRaw -check([ - 0x94, - 0xa0, - 0xa1, ?a, - 0xa2, ?b, ?c, - 0xa3, ?d, ?e, ?f, -], ["","a","bc","def"]) - -# FixMap -check([ - 0x82, - 0xc2, 0x81, - 0xc0, 0xc0, - 0xc3, 0x81, - 0xc0, 0x80, -], {false=>{nil=>nil}, true=>{nil=>{}}}) - -# unsigned int -check([ - 0x99, - 0xcc, 0, - 0xcc, 128, - 0xcc, 255, - 0xcd, 0x00, 0x00, - 0xcd, 0x80, 0x00, - 0xcd, 0xff, 0xff, - 0xce, 0x00, 0x00, 0x00, 0x00, - 0xce, 0x80, 0x00, 0x00, 0x00, - 0xce, 0xff, 0xff, 0xff, 0xff, -], [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295]) - -# signed int -check([ - 0x99, - 0xd0, 0, - 0xd0, 128, - 0xd0, 255, - 0xd1, 0x00, 0x00, - 0xd1, 0x80, 0x00, - 0xd1, 0xff, 0xff, - 0xd2, 0x00, 0x00, 0x00, 0x00, - 0xd2, 0x80, 0x00, 0x00, 0x00, - 0xd2, 0xff, 0xff, 0xff, 0xff, -], [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1]) - -# raw -check([ - 0x96, - 0xda, 0x00, 0x00, - 0xda, 0x00, 0x01, ?a, - 0xda, 0x00, 0x02, ?a, ?b, - 0xdb, 0x00, 0x00, 0x00, 0x00, - 0xdb, 0x00, 0x00, 0x00, 0x01, ?a, - 0xdb, 0x00, 0x00, 0x00, 0x02, ?a, ?b, -], ["", "a", "ab", "", "a", "ab"]) - -# array -check([ - 0x96, - 0xdc, 0x00, 0x00, - 0xdc, 0x00, 0x01, 0xc0, - 0xdc, 0x00, 0x02, 0xc2, 0xc3, - 0xdd, 0x00, 0x00, 0x00, 0x00, - 0xdd, 0x00, 0x00, 0x00, 0x01, 0xc0, - 0xdd, 0x00, 0x00, 0x00, 0x02, 0xc2, 0xc3 -], [[], [nil], [false,true], [], [nil], [false,true]]) - -# map -check([ - 0x96, - 0xde, 0x00, 0x00, - 0xde, 0x00, 0x01, 0xc0, 0xc2, - 0xde, 0x00, 0x02, 0xc0, 0xc2, 0xc3, 0xc2, - 0xdf, 0x00, 0x00, 0x00, 0x00, - 0xdf, 0x00, 0x00, 0x00, 0x01, 0xc0, 0xc2, - 0xdf, 0x00, 0x00, 0x00, 0x02, 0xc0, 0xc2, 0xc3, 0xc2, -], [{}, {nil=>false}, {true=>false, nil=>false}, {}, {nil=>false}, {true=>false, nil=>false}]) - - diff --git a/ruby/test_pack.rb b/ruby/test_pack.rb deleted file mode 100644 index 6873d57..0000000 --- a/ruby/test_pack.rb +++ /dev/null @@ -1,58 +0,0 @@ -require 'msgpack' - -def check(data) - puts "---" - pack = data.to_msgpack - p data - puts pack.unpack('C*').map{|x|"%02x"%x}.join(' ') - re = MessagePack::unpack(pack) - if re != data - p re - puts "** TEST FAILED **" - end -end - -check 0 -check 1 -check 127 -check 128 -check 255 -check 256 -check 65535 -check 65536 -check -1 -check -32 -check -33 -check -128 -check -129 -check -32768 -check -32769 - -check 1.0 - -check "" -check "a" -check "a"*31 -check "a"*32 - -check nil -check true -check false - -check [] -check [[]] -check [[], nil] - -check( {nil=>0} ) - -check (1<<23) -__END__ - -ary = [] -i = 0 -while i < (1<<16) - ary << i - i += 1 -end -check ary - From 7df178085341a5a4845d391ef7ec8317da3cc0ec Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 6 Feb 2010 21:31:16 +0900 Subject: [PATCH 0369/1172] ruby: makegem.sh --- README | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README b/README index 66ca276..6ca10d6 100644 --- a/README +++ b/README @@ -19,12 +19,12 @@ Binary-based efficient data interchange format. $ make $ sudo make install - To install Ruby binding, run ./gengem.sh script on ruby/ directory and install + To install Ruby binding, run ./makegem.sh script on ruby/ directory and install generated gem package. $ cd ruby - $ ./gengem.sh - $ gem install gem/pkg/msgpack-*.gem + $ ./makegem.sh + $ gem install msgpack-*.gem *Usage @@ -54,7 +54,7 @@ Binary-based efficient data interchange format. API Document is available at http://msgpack.sourceforge.jp/. -Copyright (C) 2008-2009 FURUHASHI Sadayuki +Copyright (C) 2008-2010 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 27787505e0897eae05ee939ad846427cf1894c99 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 27 Feb 2010 02:37:01 +0900 Subject: [PATCH 0370/1172] strict-aliasing rule 2 --- COPYING | 2 +- msgpack/pack_define.h | 3 +- msgpack/pack_template.h | 66 +++++++++++++++++++-------------------- msgpack/sysdep.h | 15 ++++++++- msgpack/unpack_define.h | 2 +- msgpack/unpack_template.h | 51 +++++++++++++----------------- 6 files changed, 72 insertions(+), 67 deletions(-) diff --git a/COPYING b/COPYING index 6f5f220..4388e8f 100644 --- a/COPYING +++ b/COPYING @@ -1,4 +1,4 @@ -Copyright (C) 2008-2009 FURUHASHI Sadayuki +Copyright (C) 2008-2010 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h index 692ef7d..4845d52 100644 --- a/msgpack/pack_define.h +++ b/msgpack/pack_define.h @@ -1,7 +1,7 @@ /* * MessagePack unpacking routine template * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ #include "msgpack/sysdep.h" #include +#include #endif /* msgpack/pack_define.h */ diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index b314d6d..493634d 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -1,7 +1,7 @@ /* * MessagePack packing routine template * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -69,7 +69,7 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -89,12 +89,12 @@ do { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -115,17 +115,17 @@ do { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ + buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -149,7 +149,7 @@ do { \ if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xd1; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -167,7 +167,7 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -179,12 +179,12 @@ do { \ if(d < -(1<<15)) { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xd2; _msgpack_store32(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xd1; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -202,12 +202,12 @@ do { \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -220,19 +220,19 @@ do { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ + buf[0] = 0xd3; _msgpack_store64(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xd2; _msgpack_store32(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xd1; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -252,19 +252,19 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ + buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -283,21 +283,21 @@ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { unsigned char buf[3]; - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); + buf[0] = 0xcd; _msgpack_store16(&buf[1], d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { unsigned char buf[5]; - buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); + buf[0] = 0xce; _msgpack_store32(&buf[1], d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { unsigned char buf[9]; - buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); + buf[0] = 0xcf; _msgpack_store64(&buf[1], d); msgpack_pack_append_buffer(x, buf, 9); } @@ -310,21 +310,21 @@ msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { unsigned char buf[3]; - buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); + buf[0] = 0xd1; _msgpack_store16(&buf[1], d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { unsigned char buf[5]; - buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); + buf[0] = 0xd2; _msgpack_store32(&buf[1], d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { unsigned char buf[9]; - buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); + buf[0] = 0xd3; _msgpack_store64(&buf[1], d); msgpack_pack_append_buffer(x, buf, 9); } @@ -557,7 +557,7 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) union { float f; uint32_t i; } mem; mem.f = d; unsigned char buf[5]; - buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(mem.i); + buf[0] = 0xca; _msgpack_store32(&buf[1], mem.i); msgpack_pack_append_buffer(x, buf, 5); } @@ -566,7 +566,7 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) union { double f; uint64_t i; } mem; mem.f = d; unsigned char buf[9]; - buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(mem.i); + buf[0] = 0xcb; _msgpack_store64(&buf[1], mem.i); msgpack_pack_append_buffer(x, buf, 9); } @@ -610,11 +610,11 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { unsigned char buf[3]; - buf[0] = 0xdc; *(uint16_t*)&buf[1] = _msgpack_be16(n); + buf[0] = 0xdc; _msgpack_store16(&buf[1], n); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdd; *(uint32_t*)&buf[1] = _msgpack_be32(n); + buf[0] = 0xdd; _msgpack_store32(&buf[1], n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -631,11 +631,11 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(n < 65536) { unsigned char buf[3]; - buf[0] = 0xde; *(uint16_t*)&buf[1] = _msgpack_be16(n); + buf[0] = 0xde; _msgpack_store16(&buf[1], n); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdf; *(uint32_t*)&buf[1] = _msgpack_be32(n); + buf[0] = 0xdf; _msgpack_store32(&buf[1], n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -652,11 +652,11 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(l < 65536) { unsigned char buf[3]; - buf[0] = 0xda; *(uint16_t*)&buf[1] = _msgpack_be16(l); + buf[0] = 0xda; _msgpack_store16(&buf[1], l); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdb; *(uint32_t*)&buf[1] = _msgpack_be32(l); + buf[0] = 0xdb; _msgpack_store32(&buf[1], l); msgpack_pack_append_buffer(x, buf, 5); } } diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h index c69c69a..2bc01c9 100644 --- a/msgpack/sysdep.h +++ b/msgpack/sysdep.h @@ -1,7 +1,7 @@ /* * MessagePack system dependencies * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -101,5 +101,18 @@ typedef unsigned int _msgpack_atomic_counter_t; #endif +#define _msgpack_store16(to, num) \ + do { uint16_t val = _msgpack_be16(num); memcpy(to, &val, 2); } while(0); +#define _msgpack_store32(to, num) \ + do { uint32_t val = _msgpack_be32(num); memcpy(to, &val, 4); } while(0); +#define _msgpack_store64(to, num) \ + do { uint64_t val = _msgpack_be64(num); memcpy(to, &val, 8); } while(0); + + +#define _msgpack_load16(cast, from) ((cast)_msgpack_be16(*(uint16_t*)from)) +#define _msgpack_load32(cast, from) ((cast)_msgpack_be32(*(uint32_t*)from)) +#define _msgpack_load64(cast, from) ((cast)_msgpack_be64(*(uint64_t*)from)) + + #endif /* msgpack/sysdep.h */ diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index c36aa51..e61f73d 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -1,7 +1,7 @@ /* * MessagePack unpacking routine template * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index cf2e74c..3f871cd 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -1,7 +1,7 @@ /* * MessagePack unpacking routine template * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -130,11 +130,6 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c #define NEXT_CS(p) \ ((unsigned int)*p & 0x1f) -#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) _msgpack_be16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) _msgpack_be32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) _msgpack_be64(*(uint64_t*)ptr) - #ifdef USE_CASE_RANGE #define SWITCH_RANGE_BEGIN switch(*p) { #define SWITCH_RANGE(FROM, TO) case FROM ... TO: @@ -223,69 +218,69 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c //case CS_ case CS_FLOAT: { union { uint32_t i; float f; } mem; - mem.i = PTR_CAST_32(n); + mem.i = _msgpack_load32(uint32_t,n); push_fixed_value(_float, mem.f); } case CS_DOUBLE: { union { uint64_t i; double f; } mem; - mem.i = PTR_CAST_64(n); + mem.i = _msgpack_load64(uint64_t,n); push_fixed_value(_double, mem.f); } case CS_UINT_8: - push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n)); + push_fixed_value(_uint8, *(uint8_t*)n); case CS_UINT_16: - push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n)); + push_fixed_value(_uint16, _msgpack_load16(uint16_t,n)); case CS_UINT_32: - push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n)); + push_fixed_value(_uint32, _msgpack_load32(uint32_t,n)); case CS_UINT_64: - push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n)); + push_fixed_value(_uint64, _msgpack_load64(uint64_t,n)); case CS_INT_8: - push_fixed_value(_int8, (int8_t)PTR_CAST_8(n)); + push_fixed_value(_int8, *(int8_t*)n); case CS_INT_16: - push_fixed_value(_int16, (int16_t)PTR_CAST_16(n)); + push_fixed_value(_int16, _msgpack_load16(int16_t,n)); case CS_INT_32: - push_fixed_value(_int32, (int32_t)PTR_CAST_32(n)); + push_fixed_value(_int32, _msgpack_load32(int32_t,n)); case CS_INT_64: - push_fixed_value(_int64, (int64_t)PTR_CAST_64(n)); + push_fixed_value(_int64, _msgpack_load64(int64_t,n)); //case CS_ //case CS_ //case CS_BIG_INT_16: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero); //case CS_BIG_INT_32: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero); //case ACS_BIG_INT_VALUE: //_big_int_zero: // // FIXME // push_variable_value(_big_int, data, n, trail); //case CS_BIG_FLOAT_16: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero); //case CS_BIG_FLOAT_32: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero); //case ACS_BIG_FLOAT_VALUE: //_big_float_zero: // // FIXME // push_variable_value(_big_float, data, n, trail); case CS_RAW_16: - again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); + again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero); case CS_RAW_32: - again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); + again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero); case ACS_RAW_VALUE: _raw_zero: push_variable_value(_raw, data, n, trail); case CS_ARRAY_16: - start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM); case CS_ARRAY_32: /* FIXME security guard */ - start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM); case CS_MAP_16: - start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY); case CS_MAP_32: /* FIXME security guard */ - start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); default: goto _failed; @@ -371,8 +366,4 @@ _end: #undef start_container #undef NEXT_CS -#undef PTR_CAST_8 -#undef PTR_CAST_16 -#undef PTR_CAST_32 -#undef PTR_CAST_64 From 05a7e4eb64f2fa616c96878d586322d869c7ff9b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 2 Mar 2010 17:09:59 +0900 Subject: [PATCH 0371/1172] ruby: add msgpack.mingw.{gemspec,sh} --- c/unpack.c | 2 +- configure.in | 2 +- ruby/makegem.sh | 5 +++++ ruby/msgpack.gemspec | 10 +++++----- ruby/msgpack.mingw.gemspec | 16 ++++++++++++++++ ruby/msgpack.mingw.sh | 21 +++++++++++++++++++++ 6 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 ruby/msgpack.mingw.gemspec create mode 100755 ruby/msgpack.mingw.sh diff --git a/c/unpack.c b/c/unpack.c index d5bcb2d..140ad1f 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -52,7 +52,7 @@ static int template_execute(template_context* ctx, static inline msgpack_object template_callback_root(unpack_user* u) -{ msgpack_object o; return o; } +{ msgpack_object o = {0}; return o; } static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_object* o) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } diff --git a/configure.in b/configure.in index 259ed37..c75957b 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.4.1) +AM_INIT_AUTOMAKE(msgpack, 0.4.2) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/ruby/makegem.sh b/ruby/makegem.sh index fd0db79..622264d 100755 --- a/ruby/makegem.sh +++ b/ruby/makegem.sh @@ -19,3 +19,8 @@ cat msgpack_test.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FI gem build msgpack.gemspec +# ./msgpack.mingw.sh # on msys +# gem push msgpack-$version.gem +# gem push msgpack-$version-x86-mingw32.gem +# gem push msgpack-$version-mswin32.gem + diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index 7273881..4575c70 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,5 +1,5 @@ Gem::Specification.new do |s| - s.platform = Gem::Platform::RUBY + s.platform = Gem::Platform::RUBY s.name = "msgpack" s.version = "0.3.3" s.summary = "MessagePack, a binary-based efficient data interchange format." @@ -7,10 +7,10 @@ Gem::Specification.new do |s| s.email = "frsyuki@users.sourceforge.jp" s.homepage = "http://msgpack.sourceforge.jp/" s.rubyforge_project = "msgpack" - s.has_rdoc = false - s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] + s.has_rdoc = false + s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] s.require_paths = ["lib", "ext"] s.files = Dir["lib/**/*", "ext/**/*", "msgpack/**/*", "test/**/*"] - s.test_files = Dir["test/test_*.rb"] - s.extensions = Dir["ext/**/extconf.rb"] + s.test_files = Dir["test/test_*.rb"] + s.extensions = Dir["ext/**/extconf.rb"] end diff --git a/ruby/msgpack.mingw.gemspec b/ruby/msgpack.mingw.gemspec new file mode 100644 index 0000000..3d2e7dc --- /dev/null +++ b/ruby/msgpack.mingw.gemspec @@ -0,0 +1,16 @@ +Gem::Specification.new do |s| + s.platform = Gem::Platform::CURRENT + s.name = "msgpack" + s.version = "0.3.3" + s.summary = "MessagePack, a binary-based efficient data interchange format." + s.author = "FURUHASHI Sadayuki" + s.email = "frsyuki@users.sourceforge.jp" + s.homepage = "http://msgpack.sourceforge.jp/" + s.rubyforge_project = "msgpack" + s.has_rdoc = false + s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] + s.require_paths = ["lib", "ext"] + s.files = Dir["lib/**/*", "ext/**/*", "msgpack/**/*", "test/**/*"] + s.test_files = Dir["test/test_*.rb"] + s.extensions = Dir["ext/**/extconf.rb"] +end diff --git a/ruby/msgpack.mingw.sh b/ruby/msgpack.mingw.sh new file mode 100755 index 0000000..ec75234 --- /dev/null +++ b/ruby/msgpack.mingw.sh @@ -0,0 +1,21 @@ +#!/bin/sh +if [ -z "$1" ];then + echo "usage: $0 " + exit 1 +fi + +version=$1 +build=msgpack-mingw-build + +./makegem.sh +gem build msgpack.mingw.gemspec +rm -rf $build +mkdir $build +cd $build +tar xvf ../msgpack-$version-x86-mingw32.gem +gunzip metadata.gz +sed s/x86-mingw32/mswin32/ metadata > metadata.tmp +mv metadata.tmp metadata +gzip metadata +tar cvf msgpack-$version-mswin32.gem metadata.gz data.tar.gz + From 568704ed2256047fbf69657b5cc1e02f76e71855 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 19 Mar 2010 13:34:53 +0900 Subject: [PATCH 0372/1172] fixes Visual C++ compatibility --- Makefile.am | 5 + c/msgpack/zone.h | 2 - c/unpack.c | 2 +- c/vrefbuffer.c | 1 - configure.in | 2 +- msgpack_vc8.postbuild.bat | 41 +++ msgpack_vc8.sln | 20 ++ msgpack_vc8.vcproj | 558 +++++++++++++++++++------------------- 8 files changed, 351 insertions(+), 280 deletions(-) create mode 100644 msgpack_vc8.postbuild.bat create mode 100644 msgpack_vc8.sln mode change 100644 => 100755 msgpack_vc8.vcproj diff --git a/Makefile.am b/Makefile.am index 42fb233..04f62d2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -11,3 +11,8 @@ nobase_include_HEADERS = \ msgpack/unpack_template.h \ msgpack/sysdep.h +EXTRA_DIST = \ + msgpack_vc8.vcproj \ + msgpack_vc8.sln \ + msgpack_vc8.postbuild.bat + diff --git a/c/msgpack/zone.h b/c/msgpack/zone.h index 475ae51..ce5be6d 100644 --- a/c/msgpack/zone.h +++ b/c/msgpack/zone.h @@ -19,8 +19,6 @@ #define MSGPACK_ZONE_H__ #include "msgpack/sysdep.h" -#include -#include #ifdef __cplusplus extern "C" { diff --git a/c/unpack.c b/c/unpack.c index 140ad1f..8be0b92 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -52,7 +52,7 @@ static int template_execute(template_context* ctx, static inline msgpack_object template_callback_root(unpack_user* u) -{ msgpack_object o = {0}; return o; } +{ msgpack_object o = {}; return o; } static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_object* o) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } diff --git a/c/vrefbuffer.c b/c/vrefbuffer.c index b528874..136372f 100644 --- a/c/vrefbuffer.c +++ b/c/vrefbuffer.c @@ -30,7 +30,6 @@ bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, vbuf->chunk_size = chunk_size; vbuf->ref_size = ref_size; - // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ size_t nfirst = (sizeof(struct iovec) < 72/2) ? 72 / sizeof(struct iovec) : 8; diff --git a/configure.in b/configure.in index c75957b..0f2c5d0 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.4.2) +AM_INIT_AUTOMAKE(msgpack, 0.4.3) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/msgpack_vc8.postbuild.bat b/msgpack_vc8.postbuild.bat new file mode 100644 index 0000000..7ee2586 --- /dev/null +++ b/msgpack_vc8.postbuild.bat @@ -0,0 +1,41 @@ +IF NOT EXIST include MKDIR include +IF NOT EXIST include\msgpack MKDIR include\msgpack +IF NOT EXIST include\msgpack\type MKDIR include\msgpack\type +IF NOT EXIST include\msgpack\type\tr1 MKDIR include\msgpack\type\tr1 +copy msgpack\pack_define.h include\msgpack\ +copy msgpack\pack_template.h include\msgpack\ +copy msgpack\unpack_define.h include\msgpack\ +copy msgpack\unpack_template.h include\msgpack\ +copy msgpack\sysdep.h include\msgpack\ +copy c\msgpack.h include\ +copy c\msgpack\sbuffer.h include\msgpack\ +copy c\msgpack\vrefbuffer.h include\msgpack\ +copy c\msgpack\pack.h include\msgpack\ +copy c\msgpack\unpack.h include\msgpack\ +copy c\msgpack\object.h include\msgpack\ +copy c\msgpack\zone.h include\msgpack\ +copy cpp\msgpack.hpp include\ +copy cpp\msgpack\sbuffer.hpp include\msgpack\ +copy cpp\msgpack\vrefbuffer.hpp include\msgpack\ +copy cpp\msgpack\pack.hpp include\msgpack\ +copy cpp\msgpack\unpack.hpp include\msgpack\ +copy cpp\msgpack\object.hpp include\msgpack\ +copy cpp\msgpack\zone.hpp include\msgpack\ +copy cpp\msgpack\type.hpp include\msgpack\type\ +copy cpp\msgpack\type\bool.hpp include\msgpack\type\ +copy cpp\msgpack\type\float.hpp include\msgpack\type\ +copy cpp\msgpack\type\int.hpp include\msgpack\type\ +copy cpp\msgpack\type\list.hpp include\msgpack\type\ +copy cpp\msgpack\type\deque.hpp include\msgpack\type\ +copy cpp\msgpack\type\map.hpp include\msgpack\type\ +copy cpp\msgpack\type\nil.hpp include\msgpack\type\ +copy cpp\msgpack\type\pair.hpp include\msgpack\type\ +copy cpp\msgpack\type\raw.hpp include\msgpack\type\ +copy cpp\msgpack\type\set.hpp include\msgpack\type\ +copy cpp\msgpack\type\string.hpp include\msgpack\type\ +copy cpp\msgpack\type\vector.hpp include\msgpack\type\ +copy cpp\msgpack\type\tuple.hpp include\msgpack\type\ +copy cpp\msgpack\type\define.hpp include\msgpack\type\ +copy cpp\msgpack\type\tr1\unordered_map.hpp include\msgpack\type\ +copy cpp\msgpack\type\tr1\unordered_set.hpp include\msgpack\type\ + diff --git a/msgpack_vc8.sln b/msgpack_vc8.sln new file mode 100644 index 0000000..84718af --- /dev/null +++ b/msgpack_vc8.sln @@ -0,0 +1,20 @@ + +Microsoft Visual Studio Solution File, Format Version 9.00 +# Visual C++ Express 2008 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "MessagePack", "msgpack_vc8.vcproj", "{122A2EA4-B283-4241-9655-786DE78283B2}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Release|Win32 = Release|Win32 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {122A2EA4-B283-4241-9655-786DE78283B2}.Debug|Win32.ActiveCfg = Debug|Win32 + {122A2EA4-B283-4241-9655-786DE78283B2}.Debug|Win32.Build.0 = Debug|Win32 + {122A2EA4-B283-4241-9655-786DE78283B2}.Release|Win32.ActiveCfg = Release|Win32 + {122A2EA4-B283-4241-9655-786DE78283B2}.Release|Win32.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/msgpack_vc8.vcproj b/msgpack_vc8.vcproj old mode 100644 new mode 100755 index 5daec01..a3fa28f --- a/msgpack_vc8.vcproj +++ b/msgpack_vc8.vcproj @@ -1,275 +1,283 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 1b1433a6642036d51e49e35b64b3534a38d95916 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 26 Mar 2010 14:29:49 +0900 Subject: [PATCH 0373/1172] ruby: copy the deserialized string if length <= RSTRING_EMBED_LEN_MAX --- ruby/unpack.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ruby/unpack.c b/ruby/unpack.c index 0f1af38..1748329 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -110,8 +110,14 @@ static inline int template_callback_map(unpack_user* u, unsigned int n, VALUE* o static inline int template_callback_map_item(unpack_user* u, VALUE* c, VALUE k, VALUE v) { rb_hash_aset(*c, k, v); return 0; } +#ifdef RSTRING_EMBED_LEN_MAX +#define COW_MIN_SIZE RSTRING_EMBED_LEN_MAX +#else +#define COW_MIN_SIZE ((sizeof(VALUE)*3)/sizeof(char)-1) +#endif + static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, VALUE* o) -{ *o = (l == 0) ? rb_str_new(0,0) : rb_str_substr(u->source, p - b, l); return 0; } +{ *o = (l <= COW_MIN_SIZE) ? rb_str_new(p, l) : rb_str_substr(u->source, p - b, l); return 0; } #include "msgpack/unpack_template.h" From 72e3f9821300f53d988e546a8dc7e001d9acb717 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 26 Mar 2010 14:30:50 +0900 Subject: [PATCH 0374/1172] cpp: alias pack(Stream* s, const T& v) -> pack(Stream& const T& v) --- c/msgpack/sbuffer.h | 3 ++- c/msgpack/vrefbuffer.h | 12 ++++++------ cpp/msgpack/pack.hpp | 10 ++++++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/c/msgpack/sbuffer.h b/c/msgpack/sbuffer.h index b0fab46..bc0a8fd 100644 --- a/c/msgpack/sbuffer.h +++ b/c/msgpack/sbuffer.h @@ -22,13 +22,14 @@ #include #ifndef MSGPACK_SBUFFER_INIT_SIZE -#define MSGPACK_SBUFFER_INIT_SIZE 2048 +#define MSGPACK_SBUFFER_INIT_SIZE 8192 #endif #ifdef __cplusplus extern "C" { #endif + typedef struct msgpack_sbuffer { size_t size; char* data; diff --git a/c/msgpack/vrefbuffer.h b/c/msgpack/vrefbuffer.h index ffc52bf..38ead67 100644 --- a/c/msgpack/vrefbuffer.h +++ b/c/msgpack/vrefbuffer.h @@ -29,12 +29,6 @@ struct iovec { }; #endif - -#ifdef __cplusplus -extern "C" { -#endif - - #ifndef MSGPACK_VREFBUFFER_REF_SIZE #define MSGPACK_VREFBUFFER_REF_SIZE 32 #endif @@ -43,6 +37,11 @@ extern "C" { #define MSGPACK_VREFBUFFER_CHUNK_SIZE 8192 #endif +#ifdef __cplusplus +extern "C" { +#endif + + struct msgpack_vrefbuffer_chunk; typedef struct msgpack_vrefbuffer_chunk msgpack_vrefbuffer_chunk; @@ -102,6 +101,7 @@ size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref) return vref->tail - vref->array; } + #ifdef __cplusplus } #endif diff --git a/cpp/msgpack/pack.hpp b/cpp/msgpack/pack.hpp index 9c291c1..b7b3ca1 100644 --- a/cpp/msgpack/pack.hpp +++ b/cpp/msgpack/pack.hpp @@ -28,6 +28,7 @@ namespace msgpack { template class packer { public: + packer(Stream* s); packer(Stream& s); ~packer(); @@ -111,6 +112,12 @@ private: }; +template +inline void pack(Stream* s, const T& v) +{ + packer(s).pack(v); +} + template inline void pack(Stream& s, const T& v) { @@ -133,6 +140,9 @@ inline void pack(Stream& s, const T& v) #include "msgpack/pack_template.h" +template +packer::packer(Stream* s) : m_stream(*s) { } + template packer::packer(Stream& s) : m_stream(s) { } From 5782ab7ccce243b1eac0fd6e10928d1aedcb552c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 26 Mar 2010 14:33:49 +0900 Subject: [PATCH 0375/1172] c,cpp: add msgpack_zbuffer and msgpack::zbuffer --- c/Makefile.am | 1 + c/msgpack/zbuffer.h | 175 ++++++++++++++++++++++++++++++++++++++++ cpp/Makefile.am | 1 + cpp/msgpack/zbuffer.hpp | 100 +++++++++++++++++++++++ 4 files changed, 277 insertions(+) create mode 100644 c/msgpack/zbuffer.h create mode 100644 cpp/msgpack/zbuffer.hpp diff --git a/c/Makefile.am b/c/Makefile.am index f84759a..bbe547c 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -10,6 +10,7 @@ nobase_include_HEADERS = \ msgpack.h \ msgpack/sbuffer.h \ msgpack/vrefbuffer.h \ + msgpack/zbuffer.h \ msgpack/pack.h \ msgpack/unpack.h \ msgpack/object.h \ diff --git a/c/msgpack/zbuffer.h b/c/msgpack/zbuffer.h new file mode 100644 index 0000000..ad3d11a --- /dev/null +++ b/c/msgpack/zbuffer.h @@ -0,0 +1,175 @@ +/* + * MessagePack for C deflate buffer implementation + * + * Copyright (C) 2010 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_ZBUFFER_H__ +#define MSGPACK_ZBUFFER_H__ + +#include "msgpack/sysdep.h" +#include +#include +#include + +#ifndef MSGPACK_ZBUFFER_INIT_SIZE +#define MSGPACK_ZBUFFER_INIT_SIZE 8192 +#endif + +#ifndef MSGPACK_ZBUFFER_RESERVE_SIZE +#define MSGPACK_ZBUFFER_RESERVE_SIZE 512 +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef struct msgpack_zbuffer { + z_stream stream; + char* data; + size_t init_size; +} msgpack_zbuffer; + + +static inline bool msgpack_zbuffer_init(msgpack_zbuffer* zbuf, + int level, size_t init_size); +static inline void msgpack_zbuffer_destroy(msgpack_zbuffer* zbuf); + +static inline char* msgpack_zbuffer_flush(msgpack_zbuffer* zbuf); + +static inline const char* msgpack_zbuffer_data(const msgpack_zbuffer* zbuf); +static inline size_t msgpack_zbuffer_size(const msgpack_zbuffer* zbuf); + +static inline bool msgpack_zbuffer_reset(msgpack_zbuffer* zbuf); +static inline void msgpack_zbuffer_reset_buffer(msgpack_zbuffer* zbuf); +static inline char* msgpack_zbuffer_release_buffer(msgpack_zbuffer* zbuf); + + +static inline int msgpack_zbuffer_write(void* data, const char* buf, unsigned int len); + +static inline bool msgpack_zbuffer_expand(msgpack_zbuffer* zbuf); + + +bool msgpack_zbuffer_init(msgpack_zbuffer* zbuf, + int level, size_t init_size) +{ + memset(zbuf, 0, sizeof(msgpack_zbuffer)); + zbuf->init_size = init_size; + if(deflateInit(&zbuf->stream, level) != Z_OK) { + free(zbuf->data); + return false; + } + return true; +} + +void msgpack_zbuffer_destroy(msgpack_zbuffer* zbuf) +{ + deflateEnd(&zbuf->stream); + free(zbuf->data); +} + +bool msgpack_zbuffer_expand(msgpack_zbuffer* zbuf) +{ + size_t used = (char*)zbuf->stream.next_out - zbuf->data; + size_t csize = used + zbuf->stream.avail_out; + size_t nsize = (csize == 0) ? zbuf->init_size : csize * 2; + + char* tmp = (char*)realloc(zbuf->data, nsize); + if(tmp == NULL) { + return false; + } + + zbuf->data = tmp; + zbuf->stream.next_out = (Bytef*)(tmp + used); + zbuf->stream.avail_out = nsize - used; + + return true; +} + +int msgpack_zbuffer_write(void* data, const char* buf, unsigned int len) +{ + msgpack_zbuffer* zbuf = (msgpack_zbuffer*)data; + + zbuf->stream.next_in = (Bytef*)buf; + zbuf->stream.avail_in = len; + + do { + if(zbuf->stream.avail_out < MSGPACK_ZBUFFER_RESERVE_SIZE) { + msgpack_zbuffer_expand(zbuf); + } + + if(deflate(&zbuf->stream, Z_NO_FLUSH) != Z_OK) { + return -1; + } + } while(zbuf->stream.avail_in > 0); + + return 0; +} + +char* msgpack_zbuffer_flush(msgpack_zbuffer* zbuf) +{ + while(true) { + switch(deflate(&zbuf->stream, Z_FINISH)) { + case Z_STREAM_END: + return zbuf->data; + case Z_OK: + msgpack_zbuffer_expand(zbuf); + default: + return NULL; + } + } +} + +const char* msgpack_zbuffer_data(const msgpack_zbuffer* zbuf) +{ + return zbuf->data; +} + +size_t msgpack_zbuffer_size(const msgpack_zbuffer* zbuf) +{ + return (char*)zbuf->stream.next_out - zbuf->data; +} + +void msgpack_zbuffer_reset_buffer(msgpack_zbuffer* zbuf) +{ + zbuf->stream.avail_out += (char*)zbuf->stream.next_out - zbuf->data; + zbuf->stream.next_out = (Bytef*)zbuf->data; +} + +bool msgpack_zbuffer_reset(msgpack_zbuffer* zbuf) +{ + if(deflateReset(&zbuf->stream) != Z_OK) { + return false; + } + msgpack_zbuffer_reset_buffer(zbuf); + return true; +} + +char* msgpack_zbuffer_release_buffer(msgpack_zbuffer* zbuf) +{ + char* tmp = zbuf->data; + zbuf->data = NULL; + zbuf->stream.next_out = NULL; + zbuf->stream.avail_out = 0; + return tmp; +} + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/zbuffer.h */ + diff --git a/cpp/Makefile.am b/cpp/Makefile.am index d4de161..c1b4981 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -7,6 +7,7 @@ nobase_include_HEADERS = \ msgpack.hpp \ msgpack/sbuffer.hpp \ msgpack/vrefbuffer.hpp \ + msgpack/zbuffer.hpp \ msgpack/pack.hpp \ msgpack/unpack.hpp \ msgpack/object.hpp \ diff --git a/cpp/msgpack/zbuffer.hpp b/cpp/msgpack/zbuffer.hpp new file mode 100644 index 0000000..278f076 --- /dev/null +++ b/cpp/msgpack/zbuffer.hpp @@ -0,0 +1,100 @@ +// +// MessagePack for C++ deflate buffer implementation +// +// Copyright (C) 2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_ZBUFFER_HPP__ +#define MSGPACK_ZBUFFER_HPP__ + +#include "msgpack/zbuffer.h" +#include + +namespace msgpack { + + +class zbuffer : public msgpack_zbuffer { +public: + zbuffer(int level = Z_DEFAULT_COMPRESSION, + size_t init_size = MSGPACK_ZBUFFER_INIT_SIZE) + { + msgpack_zbuffer_init(this, level, init_size); + } + + ~zbuffer() + { + msgpack_zbuffer_destroy(this); + } + +public: + void write(const char* buf, unsigned int len) + { + if(msgpack_zbuffer_write(this, buf, len) < 0) { + throw std::bad_alloc(); + } + } + + char* flush() + { + char* buf = msgpack_zbuffer_flush(this); + if(!buf) { + throw std::bad_alloc(); + } + return buf; + } + + char* data() + { + return base::data; + } + + const char* data() const + { + return base::data; + } + + size_t size() const + { + return msgpack_zbuffer_size(this); + } + + void reset() + { + if(!msgpack_zbuffer_reset(this)) { + throw std::bad_alloc(); + } + } + + void reset_buffer() + { + msgpack_zbuffer_reset_buffer(this); + } + + char* release_buffer() + { + return msgpack_zbuffer_release_buffer(this); + } + +private: + typedef msgpack_zbuffer base; + +private: + zbuffer(const zbuffer&); +}; + + +} // namespace msgpack + +#endif /* msgpack/zbuffer.hpp */ + From df5a60fd5b0372a636f1e104d6fd5fbb2e36c306 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 26 Mar 2010 15:16:13 +0900 Subject: [PATCH 0376/1172] ruby: append_buffer calls "<<" method if the buffer object.class != String --- ruby/pack.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/ruby/pack.c b/ruby/pack.c index 0a3711f..8e88115 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -18,6 +18,9 @@ #include "ruby.h" #include "msgpack/pack_define.h" +static ID s_to_msgpack; +static ID s_append; + #define msgpack_pack_inline_func(name) \ static inline void msgpack_pack ## name @@ -27,7 +30,9 @@ #define msgpack_pack_user VALUE #define msgpack_pack_append_buffer(user, buf, len) \ - rb_str_buf_cat(user, (const void*)buf, len) + ((TYPE(user) == T_STRING) ? \ + rb_str_buf_cat(user, (const void*)buf, len) : \ + rb_funcall(user, s_append, 1, rb_str_new((const void*)buf,len))) #include "msgpack/pack_template.h" @@ -36,8 +41,6 @@ #include "st.h" // ruby hash #endif -static ID s_to_msgpack; - #define ARG_BUFFER(name, argc, argv) \ VALUE name; \ if(argc == 1) { \ @@ -142,15 +145,24 @@ static VALUE MessagePack_Hash_to_msgpack(int argc, VALUE *argv, VALUE self) } -static VALUE MessagePack_pack(VALUE self, VALUE data) +static VALUE MessagePack_pack(int argc, VALUE* argv, VALUE self) { - return rb_funcall(data, s_to_msgpack, 0); + VALUE out; + if(argc == 1) { + out = rb_str_buf_new(0); + } else if(argc == 2) { + out = argv[1]; + } else { + rb_raise(rb_eArgError, "wrong number of arguments (%d for 1)", argc); + } + return rb_funcall(argv[0], s_to_msgpack, 1, out); } void Init_msgpack_pack(VALUE mMessagePack) { s_to_msgpack = rb_intern("to_msgpack"); + s_append = rb_intern("<<"); rb_define_method_id(rb_cNilClass, s_to_msgpack, MessagePack_NilClass_to_msgpack, -1); rb_define_method_id(rb_cTrueClass, s_to_msgpack, MessagePack_TrueClass_to_msgpack, -1); rb_define_method_id(rb_cFalseClass, s_to_msgpack, MessagePack_FalseClass_to_msgpack, -1); @@ -160,6 +172,6 @@ void Init_msgpack_pack(VALUE mMessagePack) rb_define_method_id(rb_cString, s_to_msgpack, MessagePack_String_to_msgpack, -1); rb_define_method_id(rb_cArray, s_to_msgpack, MessagePack_Array_to_msgpack, -1); rb_define_method_id(rb_cHash, s_to_msgpack, MessagePack_Hash_to_msgpack, -1); - rb_define_module_function(mMessagePack, "pack", MessagePack_pack, 1); + rb_define_module_function(mMessagePack, "pack", MessagePack_pack, -1); } From f7bdda88281812d90293901454e425d3ab282773 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 29 Mar 2010 21:18:01 +0900 Subject: [PATCH 0377/1172] cpp: fixes pack_short(int) -> pack_short(short) --- cpp/msgpack/pack.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/msgpack/pack.hpp b/cpp/msgpack/pack.hpp index b7b3ca1..6a91971 100644 --- a/cpp/msgpack/pack.hpp +++ b/cpp/msgpack/pack.hpp @@ -45,7 +45,7 @@ public: packer& pack_int32(uint32_t d); packer& pack_int64(uint64_t d); - packer& pack_short(int d); + packer& pack_short(short d); packer& pack_int(int d); packer& pack_long(long d); packer& pack_long_long(long long d); @@ -183,7 +183,7 @@ inline packer& packer::pack_int64(uint64_t d) template -inline packer& packer::pack_short(int d) +inline packer& packer::pack_short(short d) { _pack_short(m_stream, d); return *this; } template From fcce8f6d5144f7e1a3cdfb3e236cb532e5565da4 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 31 Mar 2010 12:00:26 +0900 Subject: [PATCH 0378/1172] ruby: use 'readpartial' instead of 'sysread' if !io.respond_to?(:sysread) --- ruby/pack.c | 2 +- ruby/unpack.c | 25 +++++++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/ruby/pack.c b/ruby/pack.c index 8e88115..1d0068c 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -1,7 +1,7 @@ /* * MessagePack for Ruby packing routine * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ruby/unpack.c b/ruby/unpack.c index 1748329..df54e75 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -1,7 +1,7 @@ /* * MessagePack for Ruby unpacking routine * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ #include "msgpack/unpack_define.h" static ID s_sysread; +static ID s_readpartial; typedef struct { int finished; @@ -28,6 +29,7 @@ typedef struct { VALUE buffer; VALUE stream; VALUE streambuf; + ID stream_append_method; } unpack_user; @@ -183,6 +185,15 @@ static VALUE MessagePack_Unpacker_reset(VALUE self) return self; } +static ID append_method_of(VALUE stream) +{ + if(rb_respond_to(stream, s_sysread)) { + return s_sysread; + } else { + return s_readpartial; + } +} + static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) { VALUE stream; @@ -204,6 +215,7 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) mp->user.buffer = rb_str_new("",0); mp->user.stream = stream; mp->user.streambuf = rb_str_new("",0); + mp->user.stream_append_method = append_method_of(stream); return self; } @@ -299,7 +311,9 @@ static VALUE MessagePack_Unpacker_stream_get(VALUE self) static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val) { UNPACKER(self, mp); - return mp->user.stream = val; + mp->user.stream = val; + mp->user.stream_append_method = append_method_of(val); + return val; } static VALUE MessagePack_Unpacker_fill(VALUE self) @@ -312,10 +326,12 @@ static VALUE MessagePack_Unpacker_fill(VALUE self) size_t len; if(RSTRING_LEN(mp->user.buffer) == 0) { - rb_funcall(mp->user.stream, s_sysread, 2, LONG2FIX(64*1024), mp->user.buffer); + rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, + LONG2FIX(64*1024), mp->user.buffer); len = RSTRING_LEN(mp->user.buffer); } else { - rb_funcall(mp->user.stream, s_sysread, 2, LONG2FIX(64*1024), mp->user.streambuf); + rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, + LONG2FIX(64*1024), mp->user.streambuf); len = RSTRING_LEN(mp->user.streambuf); rb_str_cat(mp->user.buffer, RSTRING_PTR(mp->user.streambuf), RSTRING_LEN(mp->user.streambuf)); } @@ -428,6 +444,7 @@ static VALUE MessagePack_unpack(VALUE self, VALUE data) void Init_msgpack_unpack(VALUE mMessagePack) { s_sysread = rb_intern("sysread"); + s_readpartial = rb_intern("readpartial"); eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); From 58201b95f2626543150c0e8738dda4babe401a64 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 31 Mar 2010 13:46:28 +0900 Subject: [PATCH 0379/1172] ruby: version 0.3.4 --- ruby/msgpack.gemspec | 4 ++-- ruby/msgpack.mingw.gemspec | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index 4575c70..fd6c6a8 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,11 +1,11 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.3.3" + s.version = "0.3.4" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" - s.homepage = "http://msgpack.sourceforge.jp/" + s.homepage = "http://msgpack.sourceforge.net/" s.rubyforge_project = "msgpack" s.has_rdoc = false s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] diff --git a/ruby/msgpack.mingw.gemspec b/ruby/msgpack.mingw.gemspec index 3d2e7dc..e7bfecb 100644 --- a/ruby/msgpack.mingw.gemspec +++ b/ruby/msgpack.mingw.gemspec @@ -1,11 +1,11 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::CURRENT s.name = "msgpack" - s.version = "0.3.3" + s.version = "0.3.4" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" - s.homepage = "http://msgpack.sourceforge.jp/" + s.homepage = "http://msgpack.sourceforge.net/" s.rubyforge_project = "msgpack" s.has_rdoc = false s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] From f91e1c17c0c22194b7a36aaa8290ec6f5e903e79 Mon Sep 17 00:00:00 2001 From: Keiji Muraishi Date: Wed, 31 Mar 2010 17:09:00 +0900 Subject: [PATCH 0380/1172] fix typo in Makefile --- python/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/Makefile b/python/Makefile index 267415c..e06794d 100644 --- a/python/Makefile +++ b/python/Makefile @@ -1,5 +1,5 @@ all: - python setup_dev.py build_ext -i -f + python setup.py build_ext -i -f python setup.py build sdist .PHONY: test From 0159084ce958aeb1ba7ffb74ded51a9e4c5bb457 Mon Sep 17 00:00:00 2001 From: Keiji Muraishi Date: Wed, 31 Mar 2010 17:09:00 +0900 Subject: [PATCH 0381/1172] fix typo in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 267415c..e06794d 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ all: - python setup_dev.py build_ext -i -f + python setup.py build_ext -i -f python setup.py build sdist .PHONY: test From 6c6df1adaf4f2b202a6241a408538d05fb2ae430 Mon Sep 17 00:00:00 2001 From: Keiji Muraishi Date: Wed, 31 Mar 2010 17:29:07 +0900 Subject: [PATCH 0382/1172] should raise TypeError on find unsupported value --- python/msgpack/_msgpack.pyx | 2 +- python/test/test_except.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 python/test/test_except.py diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index eb83c85..61ae36b 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -71,7 +71,7 @@ cdef class Packer(object): def __dealloc__(self): free(self.pk.buf); - cdef int __pack(self, object o): + cdef int __pack(self, object o) except -1: cdef long long llval cdef unsigned long long ullval cdef long longval diff --git a/python/test/test_except.py b/python/test/test_except.py new file mode 100644 index 0000000..574728f --- /dev/null +++ b/python/test/test_except.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose.tools import * +from msgpack import packs, unpacks + +import datetime + +def test_raise_on_find_unsupported_value(): + assert_raises(TypeError, packs, datetime.datetime.now()) + +if __name__ == '__main__': + from nose import main + main() From a50a83f073df4b3784d468d36d58111404505ffe Mon Sep 17 00:00:00 2001 From: Keiji Muraishi Date: Wed, 31 Mar 2010 17:29:07 +0900 Subject: [PATCH 0383/1172] should raise TypeError on find unsupported value --- msgpack/_msgpack.pyx | 2 +- test/test_except.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 test/test_except.py diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index eb83c85..61ae36b 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -71,7 +71,7 @@ cdef class Packer(object): def __dealloc__(self): free(self.pk.buf); - cdef int __pack(self, object o): + cdef int __pack(self, object o) except -1: cdef long long llval cdef unsigned long long ullval cdef long longval diff --git a/test/test_except.py b/test/test_except.py new file mode 100644 index 0000000..574728f --- /dev/null +++ b/test/test_except.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose.tools import * +from msgpack import packs, unpacks + +import datetime + +def test_raise_on_find_unsupported_value(): + assert_raises(TypeError, packs, datetime.datetime.now()) + +if __name__ == '__main__': + from nose import main + main() From 1784746e7e964a4ee4a22dfcd8dfc04ce743e404 Mon Sep 17 00:00:00 2001 From: moaikids Date: Thu, 1 Apr 2010 23:13:56 +0900 Subject: [PATCH 0384/1172] fix: CS_MAP_16 deserialize bug(line.388) --- java/src/org/msgpack/impl/UnpackerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/src/org/msgpack/impl/UnpackerImpl.java b/java/src/org/msgpack/impl/UnpackerImpl.java index ddf9940..adc62b0 100644 --- a/java/src/org/msgpack/impl/UnpackerImpl.java +++ b/java/src/org/msgpack/impl/UnpackerImpl.java @@ -385,11 +385,11 @@ public class UnpackerImpl { obj = new Object[count*2]; if(count == 0) { break _push; } // FIXME check IMapSchema //System.out.println("fixmap count:"+count); + ++top; stack_obj[top] = top_obj; stack_ct[top] = top_ct; stack_count[top] = top_count; stack_schema[top] = top_schema; - ++top; top_obj = obj; top_ct = CT_MAP_KEY; top_count = count; From 3416cf984e31dfd9706166d6113c48318ef4b08d Mon Sep 17 00:00:00 2001 From: moaikids Date: Thu, 1 Apr 2010 23:15:36 +0900 Subject: [PATCH 0385/1172] fix: org.msgpack.impl.UnpackerImpl.java CS_MAP_16 deserialize bug(line.388) --- java/build.xml | 2 +- java/src/org/msgpack/Packer.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/java/build.xml b/java/build.xml index 40d7d61..b98f7c3 100644 --- a/java/build.xml +++ b/java/build.xml @@ -13,6 +13,6 @@ - + diff --git a/java/src/org/msgpack/Packer.java b/java/src/org/msgpack/Packer.java index ebd8402..935728d 100644 --- a/java/src/org/msgpack/Packer.java +++ b/java/src/org/msgpack/Packer.java @@ -20,7 +20,6 @@ package org.msgpack; import java.io.OutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.charset.Charset; import java.util.List; import java.util.Map; From 7c863c341ed17ff737e58a3ed958a6e1314fb4de Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 2 Apr 2010 02:19:41 +0900 Subject: [PATCH 0386/1172] ruby: use gem-compile gem instead of some scripts to create binary gems --- ruby/makegem.sh | 4 +++- ruby/msgpack.gemspec | 2 +- ruby/msgpack.mingw.gemspec | 16 ---------------- ruby/msgpack.mingw.sh | 21 --------------------- 4 files changed, 4 insertions(+), 39 deletions(-) delete mode 100644 ruby/msgpack.mingw.gemspec delete mode 100755 ruby/msgpack.mingw.sh diff --git a/ruby/makegem.sh b/ruby/makegem.sh index 622264d..c78769b 100755 --- a/ruby/makegem.sh +++ b/ruby/makegem.sh @@ -19,7 +19,9 @@ cat msgpack_test.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FI gem build msgpack.gemspec -# ./msgpack.mingw.sh # on msys +# gem install gem-compile # on msys +# gem compile msgpack-$version.gem # on msys +# gem compile msgpack-$version.gem -p mswin32 # on msys # gem push msgpack-$version.gem # gem push msgpack-$version-x86-mingw32.gem # gem push msgpack-$version-mswin32.gem diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index fd6c6a8..5ff27f2 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.3.4" + s.version = "0.3.5" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" diff --git a/ruby/msgpack.mingw.gemspec b/ruby/msgpack.mingw.gemspec deleted file mode 100644 index e7bfecb..0000000 --- a/ruby/msgpack.mingw.gemspec +++ /dev/null @@ -1,16 +0,0 @@ -Gem::Specification.new do |s| - s.platform = Gem::Platform::CURRENT - s.name = "msgpack" - s.version = "0.3.4" - s.summary = "MessagePack, a binary-based efficient data interchange format." - s.author = "FURUHASHI Sadayuki" - s.email = "frsyuki@users.sourceforge.jp" - s.homepage = "http://msgpack.sourceforge.net/" - s.rubyforge_project = "msgpack" - s.has_rdoc = false - s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] - s.require_paths = ["lib", "ext"] - s.files = Dir["lib/**/*", "ext/**/*", "msgpack/**/*", "test/**/*"] - s.test_files = Dir["test/test_*.rb"] - s.extensions = Dir["ext/**/extconf.rb"] -end diff --git a/ruby/msgpack.mingw.sh b/ruby/msgpack.mingw.sh deleted file mode 100755 index ec75234..0000000 --- a/ruby/msgpack.mingw.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh -if [ -z "$1" ];then - echo "usage: $0 " - exit 1 -fi - -version=$1 -build=msgpack-mingw-build - -./makegem.sh -gem build msgpack.mingw.gemspec -rm -rf $build -mkdir $build -cd $build -tar xvf ../msgpack-$version-x86-mingw32.gem -gunzip metadata.gz -sed s/x86-mingw32/mswin32/ metadata > metadata.tmp -mv metadata.tmp metadata -gzip metadata -tar cvf msgpack-$version-mswin32.gem metadata.gz data.tar.gz - From 11286524a5872f88a0988118faaf9c37f5d0c2d7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 2 Apr 2010 03:22:29 +0900 Subject: [PATCH 0387/1172] ruby: fixes 'File not found: lib' message on gem installation --- ruby/msgpack.gemspec | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index 5ff27f2..117f058 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.3.5" + s.version = "0.3.6" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" @@ -9,8 +9,8 @@ Gem::Specification.new do |s| s.rubyforge_project = "msgpack" s.has_rdoc = false s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] - s.require_paths = ["lib", "ext"] - s.files = Dir["lib/**/*", "ext/**/*", "msgpack/**/*", "test/**/*"] + s.require_paths = ["ext"] + s.files = Dir["ext/**/*", "msgpack/**/*", "test/**/*"] s.test_files = Dir["test/test_*.rb"] s.extensions = Dir["ext/**/extconf.rb"] end From 93c3cbeaeff73686048c687fa1a4cfcfd7fc2129 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 4 Apr 2010 21:45:56 +0900 Subject: [PATCH 0388/1172] ruby fixes gemspec: require_paths = ["lib"] --- ruby/msgpack.gemspec | 2 +- ruby/test/test_helper.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index 117f058..4f7fcec 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -9,7 +9,7 @@ Gem::Specification.new do |s| s.rubyforge_project = "msgpack" s.has_rdoc = false s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] - s.require_paths = ["ext"] + s.require_paths = ["lib"] s.files = Dir["ext/**/*", "msgpack/**/*", "test/**/*"] s.test_files = Dir["test/test_*.rb"] s.extensions = Dir["ext/**/extconf.rb"] diff --git a/ruby/test/test_helper.rb b/ruby/test/test_helper.rb index 3088faa..6a63489 100644 --- a/ruby/test/test_helper.rb +++ b/ruby/test/test_helper.rb @@ -1,3 +1,3 @@ require 'test/unit' -require File.dirname(__FILE__) + '/../ext/msgpack' +require File.dirname(__FILE__) + '/../lib/msgpack' From f88c029a4c43e8d8c8bd37c34a390ffb987788ed Mon Sep 17 00:00:00 2001 From: Masahiro Nakagawa Date: Sun, 4 Apr 2010 21:55:00 +0900 Subject: [PATCH 0389/1172] c: fixes msgpack_zbuffer_flush --- c/msgpack/zbuffer.h | 1 + 1 file changed, 1 insertion(+) diff --git a/c/msgpack/zbuffer.h b/c/msgpack/zbuffer.h index ad3d11a..cfcba29 100644 --- a/c/msgpack/zbuffer.h +++ b/c/msgpack/zbuffer.h @@ -126,6 +126,7 @@ char* msgpack_zbuffer_flush(msgpack_zbuffer* zbuf) return zbuf->data; case Z_OK: msgpack_zbuffer_expand(zbuf); + break; default: return NULL; } From e43f57fe1aa97081e1794219375b4b5a42951dfc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 4 Apr 2010 22:06:27 +0900 Subject: [PATCH 0390/1172] c: fixes msgpack_zbuffer_flush: error checking --- c/msgpack/zbuffer.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/c/msgpack/zbuffer.h b/c/msgpack/zbuffer.h index cfcba29..26eb29f 100644 --- a/c/msgpack/zbuffer.h +++ b/c/msgpack/zbuffer.h @@ -125,7 +125,9 @@ char* msgpack_zbuffer_flush(msgpack_zbuffer* zbuf) case Z_STREAM_END: return zbuf->data; case Z_OK: - msgpack_zbuffer_expand(zbuf); + if(!msgpack_zbuffer_expand(zbuf)) { + return NULL; + } break; default: return NULL; From 254ee80c16b3b0ce12b461d189aa1e6302debea0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 4 Apr 2010 22:11:16 +0900 Subject: [PATCH 0391/1172] c: fixes msgpack_zbuffer_write: error checking --- c/msgpack/zbuffer.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/c/msgpack/zbuffer.h b/c/msgpack/zbuffer.h index 26eb29f..2a32206 100644 --- a/c/msgpack/zbuffer.h +++ b/c/msgpack/zbuffer.h @@ -107,7 +107,9 @@ int msgpack_zbuffer_write(void* data, const char* buf, unsigned int len) do { if(zbuf->stream.avail_out < MSGPACK_ZBUFFER_RESERVE_SIZE) { - msgpack_zbuffer_expand(zbuf); + if(!msgpack_zbuffer_expand(zbuf)) { + return -1; + } } if(deflate(&zbuf->stream, Z_NO_FLUSH) != Z_OK) { From 99a2d2859269ddf803a802c043babcd977260faf Mon Sep 17 00:00:00 2001 From: Hideyuki TAKEI Date: Mon, 5 Apr 2010 00:10:28 +0900 Subject: [PATCH 0392/1172] import MessagePack for PHP --- php/CREDITS | 1 + php/EXPERIMENTAL | 0 php/README | 0 php/bench/bench.php | 51 +++ php/config.m4 | 14 + php/config.w32 | 13 + php/msgpack.c | 656 ++++++++++++++++++++++++++++++ php/msgpack.php | 21 + php/msgpack/pack_define.h | 26 ++ php/msgpack/pack_template.h | 741 ++++++++++++++++++++++++++++++++++ php/msgpack/unpack_define.h | 129 ++++++ php/msgpack/unpack_template.h | 361 +++++++++++++++++ php/php_msgpack.h | 86 ++++ php/test_normal.php | 19 + php/test_streaming.php | 34 ++ php/tests/test_pack.phpt | 169 ++++++++ 16 files changed, 2321 insertions(+) create mode 100644 php/CREDITS create mode 100644 php/EXPERIMENTAL create mode 100644 php/README create mode 100644 php/bench/bench.php create mode 100644 php/config.m4 create mode 100644 php/config.w32 create mode 100644 php/msgpack.c create mode 100644 php/msgpack.php create mode 100644 php/msgpack/pack_define.h create mode 100644 php/msgpack/pack_template.h create mode 100644 php/msgpack/unpack_define.h create mode 100644 php/msgpack/unpack_template.h create mode 100644 php/php_msgpack.h create mode 100755 php/test_normal.php create mode 100755 php/test_streaming.php create mode 100644 php/tests/test_pack.phpt diff --git a/php/CREDITS b/php/CREDITS new file mode 100644 index 0000000..3aae276 --- /dev/null +++ b/php/CREDITS @@ -0,0 +1 @@ +msgpack diff --git a/php/EXPERIMENTAL b/php/EXPERIMENTAL new file mode 100644 index 0000000..e69de29 diff --git a/php/README b/php/README new file mode 100644 index 0000000..e69de29 diff --git a/php/bench/bench.php b/php/bench/bench.php new file mode 100644 index 0000000..95a7ca4 --- /dev/null +++ b/php/bench/bench.php @@ -0,0 +1,51 @@ + diff --git a/php/config.m4 b/php/config.m4 new file mode 100644 index 0000000..751b471 --- /dev/null +++ b/php/config.m4 @@ -0,0 +1,14 @@ +dnl $Id$ +dnl config.m4 for extension msgpack + +PHP_ARG_ENABLE(msgpack, whether to enable MessagePack support, +Make sure that the comment is aligned: +[ --enable-msgpack Enable MessagePack support]) + +if test "$PHP_MSGPACK" != "no"; then + dnl AC_DEFINE([HAVE_MSGPACK],1 ,[whether to enable MessagePack support]) + dnl AC_HEADER_STDC + + PHP_NEW_EXTENSION(msgpack, msgpack.c, $ext_shared) + dnl PHP_SUBST(MSGPACK_SHARED_LIBADD) +fi diff --git a/php/config.w32 b/php/config.w32 new file mode 100644 index 0000000..50e9bec --- /dev/null +++ b/php/config.w32 @@ -0,0 +1,13 @@ +// $Id$ +// vim:ft=javascript + +// If your extension references something external, use ARG_WITH +// ARG_WITH("msgpack", "for msgpack support", "no"); + +// Otherwise, use ARG_ENABLE +// ARG_ENABLE("msgpack", "enable msgpack support", "no"); + +if (PHP_MSGPACK != "no") { + EXTENSION("msgpack", "msgpack.c"); +} + diff --git a/php/msgpack.c b/php/msgpack.c new file mode 100644 index 0000000..b38eb18 --- /dev/null +++ b/php/msgpack.c @@ -0,0 +1,656 @@ +/* + +----------------------------------------------------------------------+ + | PHP Version 5 | + +----------------------------------------------------------------------+ + | Copyright (c) 1997-2007 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Author: Hideyuki TAKEI | + +----------------------------------------------------------------------+ +*/ + +/* $Id: header 226204 2007-01-01 19:32:10Z iliaa $ */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "php.h" +#include "php_ini.h" +#include "ext/standard/info.h" +#include "ext/standard/php_smart_str.h" +#include "php_msgpack.h" + +#define PHP_EXT_VERSION "0.01" + +#ifndef TRUE +# define TRUE 1 +# define FALSE 0 +#endif + + +/* pack */ +#include "msgpack/pack_define.h" + +#define msgpack_pack_inline_func(name) \ + static inline void msgpack_pack ## name + +#define msgpack_pack_inline_func_cint(name) \ + static inline void msgpack_pack ## name + +#define msgpack_pack_user smart_str* + +#define msgpack_pack_append_buffer(user, buf, len) \ + smart_str_appendl(user, (const void*)buf, len) + +#include "msgpack/pack_template.h" + + +/* unpack */ +#include "msgpack/unpack_define.h" + +typedef struct { + int finished; + char* source; +} unpack_user; + +#define msgpack_unpack_struct(name) \ + struct template ## name + +#define msgpack_unpack_func(ret, name) \ + ret template ## name + +#define msgpack_unpack_callback(name) \ + template_callback ## name + +#define msgpack_unpack_object zval* + +#define msgpack_unpack_user unpack_user + +struct template_context; +typedef struct template_context msgpack_unpack_t; + +static void template_init(msgpack_unpack_t* u); +static msgpack_unpack_object template_data(msgpack_unpack_t* u); +static int template_execute(msgpack_unpack_t* u, + const char* data, size_t len, size_t* off); + +ZEND_BEGIN_MODULE_GLOBALS(msgpack) + msgpack_unpack_t *global_mp; +ZEND_END_MODULE_GLOBALS(msgpack) + +#ifdef ZTS +#define MSGPACK_G(v) TSRMG(msgpack_globals_id, zend_msgpack_globals *, v) +#else +#define MSGPACK_G(v) (msgpack_globals.v) +#endif + +static inline msgpack_unpack_object template_callback_root(unpack_user* u) +{ + msgpack_unpack_object data; + ALLOC_INIT_ZVAL(data); + ZVAL_NULL(data); + return data; +} + +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, d); return 0; } + +static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, d); return 0; } + +static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, d); return 0; } + +static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, d); return 0; } + +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, (long)d); return 0; } + +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, (long)d); return 0; } + +static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, (long)d); return 0; } + +static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, d); return 0; } + +static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_DOUBLE(*o, d); return 0; } + +static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_DOUBLE(*o, d); return 0; } + +static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_NULL(*o); return 0; } + +static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_BOOL(*o, 1); return 0; } + +static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_BOOL(*o, 0); return 0;} + +static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); array_init(*o); return 0; } + +static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) +{ add_next_index_zval(*c, o); return 0; } + +static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); array_init(*o); return 0; } + +static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) +{ + switch(k->type) { + case IS_LONG: + add_index_zval(*c, Z_LVAL(*k), v); + break; + case IS_STRING: + add_assoc_zval_ex(*c, Z_STRVAL(*k), Z_STRLEN(*k)+1, v); + break; + default: + zend_error(E_WARNING, "[msgpack] (php_msgpack_decode) illegal offset type, skip this decoding"); + break; + } + return 0; +} + + +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) +{ + ALLOC_INIT_ZVAL(*o); + if (l == 0) { + ZVAL_STRINGL(*o, "", 0, 1); + } else { + ZVAL_STRINGL(*o, p, l, 1); + } + return 0; +} + +#include "msgpack/unpack_template.h" + +static PHP_GINIT_FUNCTION(msgpack); + +ZEND_DECLARE_MODULE_GLOBALS(msgpack) + +/* True global resources - no need for thread safety here */ +static int le_msgpack; + +/* {{{ msgpack_functions[] + * + * Every user visible function must have an entry in msgpack_functions[]. + */ +zend_function_entry msgpack_functions[] = { + PHP_FE(msgpack_pack, NULL) + PHP_FE(msgpack_unpack, NULL) + PHP_FE(msgpack_unpack_limit, NULL) + PHP_ME(msgpack, initialize, NULL, 0) + PHP_ME(msgpack, execute, NULL, 0) + PHP_ME(msgpack, execute_limit, NULL, 0) + PHP_ME(msgpack, finished, NULL, 0) + PHP_ME(msgpack, data, NULL, 0) + {NULL, NULL, NULL} /* Must be the last line in msgpack_functions[] */ +}; +/* }}} */ + +/* {{{ msgpack_module_entry + */ +zend_module_entry msgpack_module_entry = { +#if ZEND_MODULE_API_NO >= 20010901 + STANDARD_MODULE_HEADER, +#endif + "msgpack", + msgpack_functions, + PHP_MINIT(msgpack), + PHP_MSHUTDOWN(msgpack), + PHP_RINIT(msgpack), /* Replace with NULL if there's nothing to do at request start */ + PHP_RSHUTDOWN(msgpack), /* Replace with NULL if there's nothing to do at request end */ + PHP_MINFO(msgpack), +#if ZEND_MODULE_API_NO >= 20010901 + "0.1", /* Replace with version number for your extension */ +#endif + PHP_MODULE_GLOBALS(msgpack), + PHP_GINIT(msgpack), + NULL, + NULL, + STANDARD_MODULE_PROPERTIES_EX +}; +/* }}} */ + +#ifdef COMPILE_DL_MSGPACK +ZEND_GET_MODULE(msgpack) +#endif + +/* {{{ PHP_GINIT_FUNCTION */ +static PHP_GINIT_FUNCTION(msgpack) +{ + msgpack_globals->global_mp = NULL; +} +/* }}} */ + +/* {{{ PHP_MINIT_FUNCTION + */ +PHP_MINIT_FUNCTION(msgpack) +{ + zend_class_entry ce; + INIT_CLASS_ENTRY(ce, "MessagePack", msgpack_functions); + msgpack_ce = zend_register_internal_class(&ce TSRMLS_CC); + + return SUCCESS; +} +/* }}} */ + +/* {{{ PHP_MSHUTDOWN_FUNCTION + */ +PHP_MSHUTDOWN_FUNCTION(msgpack) +{ + /* uncomment this line if you have INI entries + UNREGISTER_INI_ENTRIES(); + */ + if (MSGPACK_G(global_mp)) { + efree(MSGPACK_G(global_mp)); + MSGPACK_G(global_mp) = NULL; + } + + return SUCCESS; +} +/* }}} */ + +/* Remove if there's nothing to do at request start */ +/* {{{ PHP_RINIT_FUNCTION + */ +PHP_RINIT_FUNCTION(msgpack) +{ + return SUCCESS; +} +/* }}} */ + +/* Remove if there's nothing to do at request end */ +/* {{{ PHP_RSHUTDOWN_FUNCTION + */ +PHP_RSHUTDOWN_FUNCTION(msgpack) +{ + return SUCCESS; +} +/* }}} */ + +/* {{{ PHP_MINFO_FUNCTION + */ +PHP_MINFO_FUNCTION(msgpack) +{ + php_info_print_table_start(); + php_info_print_table_header(2, "msgpack support", "enabled"); + php_info_print_table_row(2, "php extension version", PHP_EXT_VERSION); + php_info_print_table_row(2, "author", "Hideyuki TAKEI"); + php_info_print_table_row(2, "homepage", "http://msgpack.sourceforge.net"); + php_info_print_table_row(2, "open sourced by", "KLab inc."); + php_info_print_table_end(); +} +/* }}} */ + +PHP_MSGPACK_API int msgpack_determine_array_type(zval **val TSRMLS_DC) /* {{{ */ +{ + int i; + HashTable *myht = HASH_OF(*val); + + i = myht ? zend_hash_num_elements(myht) : 0; + if (i > 0) { + char *key; + ulong index, idx; + uint key_len; + HashPosition pos; + + zend_hash_internal_pointer_reset_ex(myht, &pos); + idx = 0; + for (;; zend_hash_move_forward_ex(myht, &pos)) { + i = zend_hash_get_current_key_ex(myht, &key, &key_len, &index, 0, &pos); + if (i == HASH_KEY_NON_EXISTANT) + break; + if (i == HASH_KEY_IS_STRING) { + return 1; + } else { + if (index != idx) { + return 1; + } + } + idx++; + } + } + return 0; +} +/* }}} */ + +PHP_MSGPACK_API void msgpack_pack_array_hash(smart_str *pk, zval **val TSRMLS_DC) /* {{{ */ +{ + int i, r; + HashTable *myht; + + if(Z_TYPE_PP(val) == IS_ARRAY){ + myht = HASH_OF(*val); + r = msgpack_determine_array_type(val TSRMLS_CC); + } + else{ + myht = Z_OBJPROP_PP(val); + r = 1; + } + + i = myht ? zend_hash_num_elements(myht) : 0; + + if(r == 0){ + msgpack_pack_array(pk, i); + } + else{ + msgpack_pack_map(pk, i); + } + + if(i>0){ + char *key; + zval **data; + ulong index; + uint key_len; + HashPosition pos; + HashTable *tmp_ht; + int need_comma = 0; + + zend_hash_internal_pointer_reset_ex(myht, &pos); + for(;; zend_hash_move_forward_ex(myht, &pos)){ + i = zend_hash_get_current_key_ex(myht, &key, &key_len, &index, 0, &pos); + if(i==HASH_KEY_NON_EXISTANT) + break; + if(zend_hash_get_current_data_ex(myht, (void **) &data, &pos) == SUCCESS){ + tmp_ht = HASH_OF(*data); + if (tmp_ht) + tmp_ht->nApplyCount++; + + if(r==0) + php_msgpack_pack(pk, *data TSRMLS_CC); + else if(r==1){ + if(i==HASH_KEY_IS_STRING){ + if(key[0]=='\0' && Z_TYPE_PP(val)==IS_OBJECT){ + // Skip protected and private members. + if(tmp_ht) + tmp_ht->nApplyCount--; + continue; + } + msgpack_pack_raw(pk, key_len-1); + msgpack_pack_raw_body(pk, key, key_len-1); + php_msgpack_pack(pk, *data TSRMLS_CC); + } + else{ + msgpack_pack_long(pk, index); + php_msgpack_pack(pk, *data TSRMLS_CC); + } + } + + if(tmp_ht){ + tmp_ht->nApplyCount--; + } + } + } + + } +} +/* }}} */ + +PHP_MSGPACK_API void php_msgpack_pack(smart_str *pk, zval *val TSRMLS_DC) /* {{{ */ +{ + switch(Z_TYPE_P(val)){ + case IS_NULL: + msgpack_pack_nil(pk); + break; + case IS_BOOL: + if (Z_BVAL_P(val)) + msgpack_pack_true(pk); + else + msgpack_pack_false(pk); + break; + case IS_LONG: + msgpack_pack_long(pk, Z_LVAL_P(val)); + break; + case IS_DOUBLE: + { + double dbl = Z_DVAL_P(val); + if (zend_isinf(dbl) || zend_isnan(dbl)) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_pack) double %.9g does not conform to the MSGPACK spec, encoded as 0", dbl); + ZVAL_LONG(val, 0); + } + msgpack_pack_double(pk, Z_DVAL_P(val)); + } + break; + case IS_STRING: + msgpack_pack_raw(pk, Z_STRLEN_P(val)); + msgpack_pack_raw_body(pk, Z_STRVAL_P(val), Z_STRLEN_P(val)); + break; + case IS_ARRAY: + case IS_OBJECT: + msgpack_pack_array_hash(pk, &val TSRMLS_CC); + break; + defalut: + zend_error(E_WARNING, "[msgpack] (php_msgpack_pack) type is unsupported, encoded as null"); + msgpack_pack_nil(pk); + break; + } + + return; +} +/* }}} */ + +PHP_MSGPACK_API void php_msgpack_unpack_limit(zval *return_value, const char *buf, int len, zend_bool assoc TSRMLS_DC) /* {{{ */ +{ + if (len<=0) { + RETURN_NUL(); + } + + msgpack_unpack_t mp; + template_init(&mp); + unpack_user u = {0, ""}; + + size_t from = 0; + char* dptr = (char*)buf; + long dlen = len; + int ret; + + (&mp)->user.source = (char*)buf; + ret = template_execute(&mp, dptr, (size_t)dlen, &from); + (&mp)->user.source = ""; + + if(ret < 0) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_unpack) parse error"); + } else if(ret == 0) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_unpack) insufficient bytes"); + } else { + if(from < dlen) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_unpack) extra bytes"); + } + + *return_value = *template_data(&mp); + FREE_ZVAL(template_data(&mp)); + } +} +/* }}} */ + + +PHP_FUNCTION(msgpack_pack) +{ + zval *parameter; + smart_str buf = {0}; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "z", ¶meter) == FAILURE) { + return; + } + + php_msgpack_pack(&buf, parameter TSRMLS_CC); + + ZVAL_STRINGL(return_value, buf.c, buf.len, 1); + + smart_str_free(&buf); +} + +PHP_FUNCTION(msgpack_unpack) +{ + char *parameter; + int parameter_len; + zend_bool assoc = 0; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|b", + ¶meter, ¶meter_len, &assoc) == FAILURE) { + return; + } + + if (!parameter_len) { + RETURN_NULL(); + } + + php_msgpack_unpack_limit(return_value, parameter, parameter_len, assoc TSRMLS_CC); +} + +PHP_FUNCTION(msgpack_unpack_limit) +{ + char *parameter; + int parameter_len; + int limit; + zend_bool assoc = 0; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "sl|b", + ¶meter, ¶meter_len, &limit, &assoc) == FAILURE) { + return; + } + + if (!parameter_len) { + RETURN_NULL(); + } + else if (parameter_len < limit) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_unpack) limit greater than data_len"); + limit = parameter_len; + } + + php_msgpack_unpack_limit(return_value, parameter, limit, assoc TSRMLS_CC); +} + + +PHP_MSGPACK_API void php_msgpack_unpacker_execute_limit(zval *return_value, const char *buf, int off, int len, zend_bool assoc TSRMLS_DC) /* {{{ */ +{ + if (len<=0) { + RETURN_NUL(); + } + + size_t from = off; + char* dptr = (char*)buf; + long dlen = len; + int ret; + + if(from >= dlen) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_unpacker_execute_limit) offset is bigger than data buffer size"); + } + + MSGPACK_G(global_mp)->user.source = (char*)buf; + ret = template_execute(MSGPACK_G(global_mp), dptr, (size_t)dlen, &from); + MSGPACK_G(global_mp)->user.source = ""; + + if(ret < 0) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_unpacker_execute_limit) parse error"); + } else if(ret > 0) { + MSGPACK_G(global_mp)->user.finished = 1; + RETVAL_LONG(from); + } else { + MSGPACK_G(global_mp)->user.finished = 0; + RETVAL_LONG(from); + } +} +/* }}} */ + +PHP_MSGPACK_API void php_msgpack_unpacker_reset(TSRMLS_D) /* {{{ */ +{ + if(MSGPACK_G(global_mp)) { + efree(MSGPACK_G(global_mp)); + MSGPACK_G(global_mp) = NULL; + } + MSGPACK_G(global_mp) = safe_emalloc(sizeof(msgpack_unpack_t), 1, 0); + + template_init(MSGPACK_G(global_mp)); + unpack_user u = {0, ""}; + MSGPACK_G(global_mp)->user = u; + return; +} +/* }}} */ + +PHP_METHOD(msgpack, initialize) +{ + php_msgpack_unpacker_reset(TSRMLS_C); + return; +} + +PHP_METHOD(msgpack, execute) +{ + char *data; + int off; + int data_len; + zend_bool assoc = 0; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "sl|b", + &data, &data_len, &off, &assoc) == FAILURE) { + return; + } + + if (!data_len) { + RETURN_NULL(); + } + + php_msgpack_unpacker_execute_limit(return_value, data, off, data_len, assoc TSRMLS_CC); +} + +PHP_METHOD(msgpack, execute_limit) +{ + char *data; + int off; + int data_len; + int limit; + zend_bool assoc = 0; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "sll|b", + &data, &data_len, &off, &limit, &assoc) == FAILURE) { + return; + } + + if (!data_len) { + RETURN_NULL(); + } + else if (data_len < limit) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_unpack) limit greater than (data+off)_len"); + limit = data_len; + } + + php_msgpack_unpacker_execute_limit(return_value, data, off, limit, assoc TSRMLS_CC); +} + +PHP_METHOD(msgpack, finished) +{ + if(MSGPACK_G(global_mp)->user.finished == 1) { + RETURN_TRUE; + } + RETURN_FALSE; +} + +PHP_METHOD(msgpack, data) +{ + *return_value = *template_data(MSGPACK_G(global_mp)); + return; +} + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * End: + * vim600: noet sw=4 ts=4 fdm=marker + * vim<600: noet sw=4 ts=4 + */ diff --git a/php/msgpack.php b/php/msgpack.php new file mode 100644 index 0000000..c2498dd --- /dev/null +++ b/php/msgpack.php @@ -0,0 +1,21 @@ +"; + +if(!extension_loaded('msgpack')) { + dl('msgpack.' . PHP_SHLIB_SUFFIX); +} +$module = 'msgpack'; +$functions = get_extension_funcs($module); +echo "Functions available in the test extension:$br\n"; +foreach($functions as $func) { + echo $func."$br\n"; +} +echo "$br\n"; +$function = 'confirm_' . $module . '_compiled'; +if (extension_loaded($module)) { + $str = $function($module); +} else { + $str = "Module $module is not compiled into PHP"; +} +echo "$str\n"; +?> diff --git a/php/msgpack/pack_define.h b/php/msgpack/pack_define.h new file mode 100644 index 0000000..33408e5 --- /dev/null +++ b/php/msgpack/pack_define.h @@ -0,0 +1,26 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_DEFINE_H__ +#define MSGPACK_PACK_DEFINE_H__ + +#include +#include +#include + +#endif /* msgpack/pack_define.h */ + diff --git a/php/msgpack/pack_template.h b/php/msgpack/pack_template.h new file mode 100644 index 0000000..aa620f5 --- /dev/null +++ b/php/msgpack/pack_template.h @@ -0,0 +1,741 @@ +/* + * MessagePack packing routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + + +#ifdef __LITTLE_ENDIAN__ + +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] + + +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#elif __BIG_ENDIAN__ + +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] + + +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[1] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[3] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[7] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#endif + +#ifndef msgpack_pack_inline_func +#error msgpack_pack_inline_func template is not defined +#endif + +#ifndef msgpack_pack_user +#error msgpack_pack_user type is not defined +#endif + +#ifndef msgpack_pack_append_buffer +#error msgpack_pack_append_buffer callback is not defined +#endif + + +/* + * Integer + */ + +#define msgpack_pack_real_uint8(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ +} while(0) + +#define msgpack_pack_real_uint16(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + } else if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ +} while(0) + +#define msgpack_pack_real_uint32(x, d) \ +do { \ + if(d < (1<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_uint64(x, d) \ +do { \ + if(d < (1ULL<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1ULL<<16)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else if(d < (1ULL<<32)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int8(x, d) \ +do { \ + if(d < -(1<<5)) { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + } \ +} while(0) + +#define msgpack_pack_real_int16(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int32(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<15)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int64(x, d) \ +do { \ + if(d < -(1LL<<5)) { \ + if(d < -(1LL<<15)) { \ + if(d < -(1LL<<31)) { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } else { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } else { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + } else { \ + if(d < (1LL<<16)) { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } else { \ + if(d < (1LL<<32)) { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* unsigned 64 */ \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ + } \ +} while(0) + + +#ifdef msgpack_pack_inline_func_fastint + +msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) +{ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; + msgpack_pack_append_buffer(x, buf, 2); +} + +msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) +{ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) +{ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) +{ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) +{ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; + msgpack_pack_append_buffer(x, buf, 2); +} + +msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) +{ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) +{ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) +{ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +#undef msgpack_pack_inline_func_fastint +#endif + + +msgpack_pack_inline_func(_uint8)(msgpack_pack_user x, uint8_t d) +{ + msgpack_pack_real_uint8(x, d); +} + +msgpack_pack_inline_func(_uint16)(msgpack_pack_user x, uint16_t d) +{ + msgpack_pack_real_uint16(x, d); +} + +msgpack_pack_inline_func(_uint32)(msgpack_pack_user x, uint32_t d) +{ + msgpack_pack_real_uint32(x, d); +} + +msgpack_pack_inline_func(_uint64)(msgpack_pack_user x, uint64_t d) +{ + msgpack_pack_real_uint64(x, d); +} + +msgpack_pack_inline_func(_int8)(msgpack_pack_user x, int8_t d) +{ + msgpack_pack_real_int8(x, d); +} + +msgpack_pack_inline_func(_int16)(msgpack_pack_user x, int16_t d) +{ + msgpack_pack_real_int16(x, d); +} + +msgpack_pack_inline_func(_int32)(msgpack_pack_user x, int32_t d) +{ + msgpack_pack_real_int32(x, d); +} + +msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d) +{ + msgpack_pack_real_int64(x, d); +} + + +#ifdef msgpack_pack_inline_func_cint + +msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d) +{ +#if defined(SIZEOF_SHORT) || defined(SHRT_MAX) +#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(short) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(short) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d) +{ +#if defined(SIZEOF_INT) || defined(INT_MAX) +#if SIZEOF_INT == 2 || INT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(int) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(int) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d) +{ +#if defined(SIZEOF_LONG) || defined(LONG_MAX) +#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d) +{ +#if defined(SIZEOF_SHORT) || defined(USHRT_MAX) +#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned short) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned short) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d) +{ +#if defined(SIZEOF_INT) || defined(UINT_MAX) +#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d) +{ +#if defined(SIZEOF_LONG) || defined(ULONG_MAX) +#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned long long) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned long long) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +#undef msgpack_pack_inline_func_cint +#endif + + + +/* + * Float + */ + +msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) +{ + union { char buf[4]; uint32_t num; } f; + *((float*)&f.buf) = d; // FIXME + const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) +{ + union { char buf[8]; uint64_t num; } f; + *((double*)&f.buf) = d; // FIXME + const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; + msgpack_pack_append_buffer(x, buf, 9); +} + + +/* + * Nil + */ + +msgpack_pack_inline_func(_nil)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc0; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Boolean + */ + +msgpack_pack_inline_func(_true)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc3; + msgpack_pack_append_buffer(x, &d, 1); +} + +msgpack_pack_inline_func(_false)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc2; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Array + */ + +msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x90 | n; + msgpack_pack_append_buffer(x, &d, 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Map + */ + +msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x80 | n; + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xde, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Raw + */ + +msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) +{ + if(l < 32) { + unsigned char d = 0xa0 | l; + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + } else if(l < 65536) { + uint16_t d = (uint16_t)l; + unsigned char buf[3] = {0xda, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)l; + unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + +msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l) +{ + msgpack_pack_append_buffer(x, (const unsigned char*)b, l); +} + +#undef msgpack_pack_inline_func +#undef msgpack_pack_user +#undef msgpack_pack_append_buffer + +#undef STORE8_BE8 + +#undef STORE16_BE8 +#undef STORE16_BE16 + +#undef STORE32_BE8 +#undef STORE32_BE16 +#undef STORE32_BE32 + +#undef STORE64_BE8 +#undef STORE64_BE16 +#undef STORE64_BE32 +#undef STORE64_BE64 + +#undef msgpack_pack_real_uint8 +#undef msgpack_pack_real_uint16 +#undef msgpack_pack_real_uint32 +#undef msgpack_pack_real_uint64 +#undef msgpack_pack_real_int8 +#undef msgpack_pack_real_int16 +#undef msgpack_pack_real_int32 +#undef msgpack_pack_real_int64 + diff --git a/php/msgpack/unpack_define.h b/php/msgpack/unpack_define.h new file mode 100644 index 0000000..63668c2 --- /dev/null +++ b/php/msgpack/unpack_define.h @@ -0,0 +1,129 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_UNPACK_DEFINE_H__ +#define MSGPACK_UNPACK_DEFINE_H__ + +#include +#include +#include +#include +#include +#ifndef __WIN32__ +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifndef MSGPACK_MAX_STACK_SIZE +#define MSGPACK_MAX_STACK_SIZE 16 +#endif + + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#define msgpack_betoh16(x) ntohs(x) +#define msgpack_betoh32(x) ntohl(x) + +#ifdef __LITTLE_ENDIAN__ +#if defined(__bswap_64) +# define msgpack_betoh64(x) __bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) +#else +static inline uint64_t msgpack_betoh64(uint64_t x) { + return ((x << 56) & 0xff00000000000000ULL ) | + ((x << 40) & 0x00ff000000000000ULL ) | + ((x << 24) & 0x0000ff0000000000ULL ) | + ((x << 8) & 0x000000ff00000000ULL ) | + ((x >> 8) & 0x00000000ff000000ULL ) | + ((x >> 24) & 0x0000000000ff0000ULL ) | + ((x >> 40) & 0x000000000000ff00ULL ) | + ((x >> 56) & 0x00000000000000ffULL ) ; +} +#endif +#else +#define msgpack_betoh64(x) (x) +#endif + + +typedef enum { + CS_HEADER = 0x00, // nil + + //CS_ = 0x01, + //CS_ = 0x02, // false + //CS_ = 0x03, // true + + //CS_ = 0x04, + //CS_ = 0x05, + //CS_ = 0x06, + //CS_ = 0x07, + + //CS_ = 0x08, + //CS_ = 0x09, + CS_FLOAT = 0x0a, + CS_DOUBLE = 0x0b, + CS_UINT_8 = 0x0c, + CS_UINT_16 = 0x0d, + CS_UINT_32 = 0x0e, + CS_UINT_64 = 0x0f, + CS_INT_8 = 0x10, + CS_INT_16 = 0x11, + CS_INT_32 = 0x12, + CS_INT_64 = 0x13, + + //CS_ = 0x14, + //CS_ = 0x15, + //CS_BIG_INT_16 = 0x16, + //CS_BIG_INT_32 = 0x17, + //CS_BIG_FLOAT_16 = 0x18, + //CS_BIG_FLOAT_32 = 0x19, + CS_RAW_16 = 0x1a, + CS_RAW_32 = 0x1b, + CS_ARRAY_16 = 0x1c, + CS_ARRAY_32 = 0x1d, + CS_MAP_16 = 0x1e, + CS_MAP_32 = 0x1f, + + //ACS_BIG_INT_VALUE, + //ACS_BIG_FLOAT_VALUE, + ACS_RAW_VALUE, +} msgpack_unpack_state; + + +typedef enum { + CT_ARRAY_ITEM, + CT_MAP_KEY, + CT_MAP_VALUE, +} msgpack_container_type; + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/unpack_define.h */ + diff --git a/php/msgpack/unpack_template.h b/php/msgpack/unpack_template.h new file mode 100644 index 0000000..d67fd1e --- /dev/null +++ b/php/msgpack/unpack_template.h @@ -0,0 +1,361 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef msgpack_unpack_func +#error msgpack_unpack_func template is not defined +#endif + +#ifndef msgpack_unpack_callback +#error msgpack_unpack_callback template is not defined +#endif + +#ifndef msgpack_unpack_struct +#error msgpack_unpack_struct template is not defined +#endif + +#ifndef msgpack_unpack_struct_decl +#define msgpack_unpack_struct_decl(name) msgpack_unpack_struct(name) +#endif + +#ifndef msgpack_unpack_object +#error msgpack_unpack_object type is not defined +#endif + +#ifndef msgpack_unpack_user +#error msgpack_unpack_user type is not defined +#endif + + +msgpack_unpack_struct_decl(_stack) { + msgpack_unpack_object obj; + size_t count; + unsigned int ct; + msgpack_unpack_object map_key; +}; + +msgpack_unpack_struct_decl(_context) { + msgpack_unpack_user user; + unsigned int cs; + unsigned int trail; + unsigned int top; + msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE]; +}; + + +msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) +{ + ctx->cs = CS_HEADER; + ctx->trail = 0; + ctx->top = 0; + ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); +} + +msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) +{ + return (ctx)->stack[0].obj; +} + + +msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) +{ + assert(len >= *off); + + const unsigned char* p = (unsigned char*)data + *off; + const unsigned char* const pe = (unsigned char*)data + len; + const void* n = NULL; + + unsigned int trail = ctx->trail; + unsigned int cs = ctx->cs; + unsigned int top = ctx->top; + msgpack_unpack_struct(_stack)* stack = ctx->stack; + msgpack_unpack_user* user = &ctx->user; + + msgpack_unpack_object obj; + msgpack_unpack_struct(_stack)* c = NULL; + + int ret; + +#define push_simple_value(func) \ + if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \ + goto _push +#define push_fixed_value(func, arg) \ + if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \ + goto _push +#define push_variable_value(func, base, pos, len) \ + if(msgpack_unpack_callback(func)(user, \ + (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ + goto _push + +#define again_fixed_trail(_cs, trail_len) \ + trail = trail_len; \ + cs = _cs; \ + goto _fixed_trail_again +#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ + trail = trail_len; \ + if(trail == 0) { goto ifzero; } \ + cs = _cs; \ + goto _fixed_trail_again + +#define start_container(func, count_, ct_) \ + if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ + if((count_) == 0) { obj = stack[top].obj; goto _push; } \ + if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ + stack[top].ct = ct_; \ + stack[top].count = count_; \ + /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ + /*printf("stack push %d\n", top);*/ \ + ++top; \ + goto _header_again + +#define NEXT_CS(p) \ + ((unsigned int)*p & 0x1f) + +#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) +#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) + + if(p == pe) { goto _out; } + do { + switch(cs) { + case CS_HEADER: + switch(*p) { + case 0x00 ... 0x7f: // Positive Fixnum + push_fixed_value(_uint8, *(uint8_t*)p); + case 0xe0 ... 0xff: // Negative Fixnum + push_fixed_value(_int8, *(int8_t*)p); + case 0xc0 ... 0xdf: // Variable + switch(*p) { + case 0xc0: // nil + push_simple_value(_nil); + //case 0xc1: // string + // again_terminal_trail(NEXT_CS(p), p+1); + case 0xc2: // false + push_simple_value(_false); + case 0xc3: // true + push_simple_value(_true); + //case 0xc4: + //case 0xc5: + //case 0xc6: + //case 0xc7: + //case 0xc8: + //case 0xc9: + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); + //case 0xd4: + //case 0xd5: + //case 0xd6: // big integer 16 + //case 0xd7: // big integer 32 + //case 0xd8: // big float 16 + //case 0xd9: // big float 32 + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); + default: + goto _failed; + } + case 0xa0 ... 0xbf: // FixRaw + again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); + case 0x90 ... 0x9f: // FixArray + start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); + case 0x80 ... 0x8f: // FixMap + start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); + + default: + goto _failed; + } + // end CS_HEADER + + + _fixed_trail_again: + ++p; + + default: + if((size_t)(pe - p) < trail) { goto _out; } + n = p; p += trail - 1; + switch(cs) { + //case CS_ + //case CS_ + case CS_FLOAT: { + union { uint32_t num; char buf[4]; } f; + f.num = PTR_CAST_32(n); // FIXME + push_fixed_value(_float, *((float*)f.buf)); } + case CS_DOUBLE: { + union { uint64_t num; char buf[8]; } f; + f.num = PTR_CAST_64(n); // FIXME + push_fixed_value(_double, *((double*)f.buf)); } + case CS_UINT_8: + push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n)); + case CS_UINT_16: + push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n)); + case CS_UINT_32: + push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n)); + case CS_UINT_64: + push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n)); + + case CS_INT_8: + push_fixed_value(_int8, (int8_t)PTR_CAST_8(n)); + case CS_INT_16: + push_fixed_value(_int16, (int16_t)PTR_CAST_16(n)); + case CS_INT_32: + push_fixed_value(_int32, (int32_t)PTR_CAST_32(n)); + case CS_INT_64: + push_fixed_value(_int64, (int64_t)PTR_CAST_64(n)); + + //case CS_ + //case CS_ + //case CS_BIG_INT_16: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); + //case CS_BIG_INT_32: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); + //case ACS_BIG_INT_VALUE: + //_big_int_zero: + // // FIXME + // push_variable_value(_big_int, data, n, trail); + + //case CS_BIG_FLOAT_16: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); + //case CS_BIG_FLOAT_32: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); + //case ACS_BIG_FLOAT_VALUE: + //_big_float_zero: + // // FIXME + // push_variable_value(_big_float, data, n, trail); + + case CS_RAW_16: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); + case CS_RAW_32: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); + case ACS_RAW_VALUE: + _raw_zero: + push_variable_value(_raw, data, n, trail); + + case CS_ARRAY_16: + start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + case CS_ARRAY_32: + /* FIXME security guard */ + start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + + case CS_MAP_16: + start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + case CS_MAP_32: + /* FIXME security guard */ + start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + + default: + goto _failed; + } + } + +_push: + if(top == 0) { goto _finish; } + c = &stack[top-1]; + switch(c->ct) { + case CT_ARRAY_ITEM: + if(msgpack_unpack_callback(_array_item)(user, &c->obj, obj) < 0) { goto _failed; } + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + goto _header_again; + case CT_MAP_KEY: + c->map_key = obj; + c->ct = CT_MAP_VALUE; + goto _header_again; + case CT_MAP_VALUE: + if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + c->ct = CT_MAP_KEY; + goto _header_again; + + default: + goto _failed; + } + +_header_again: + cs = CS_HEADER; + ++p; + } while(p != pe); + goto _out; + + +_finish: + stack[0].obj = obj; + ++p; + ret = 1; + /*printf("-- finish --\n"); */ + goto _end; + +_failed: + /*printf("** FAILED **\n"); */ + ret = -1; + goto _end; + +_out: + ret = 0; + goto _end; + +_end: + ctx->cs = cs; + ctx->trail = trail; + ctx->top = top; + *off = p - (const unsigned char*)data; + + return ret; +} + + +#undef msgpack_unpack_func +#undef msgpack_unpack_callback +#undef msgpack_unpack_struct +#undef msgpack_unpack_object +#undef msgpack_unpack_user + +#undef push_simple_value +#undef push_fixed_value +#undef push_variable_value +#undef again_fixed_trail +#undef again_fixed_trail_if_zero +#undef start_container + +#undef NEXT_CS +#undef PTR_CAST_8 +#undef PTR_CAST_16 +#undef PTR_CAST_32 +#undef PTR_CAST_64 + diff --git a/php/php_msgpack.h b/php/php_msgpack.h new file mode 100644 index 0000000..a1f434c --- /dev/null +++ b/php/php_msgpack.h @@ -0,0 +1,86 @@ +/* + +----------------------------------------------------------------------+ + | PHP Version 5 | + +----------------------------------------------------------------------+ + | Copyright (c) 1997-2007 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Author: | + +----------------------------------------------------------------------+ +*/ + +/* $Id: header 226204 2007-01-01 19:32:10Z iliaa $ */ + +#ifndef PHP_MSGPACK_H +#define PHP_MSGPACK_H + +extern zend_module_entry msgpack_module_entry; +#define phpext_msgpack_ptr &msgpack_module_entry + +#ifdef PHP_WIN32 +#define PHP_MSGPACK_API __declspec(dllexport) +#else +#define PHP_MSGPACK_API +#endif + +#ifdef ZTS +#include "TSRM.h" +#endif + +PHP_MINIT_FUNCTION(msgpack); +PHP_MSHUTDOWN_FUNCTION(msgpack); +PHP_RINIT_FUNCTION(msgpack); +PHP_RSHUTDOWN_FUNCTION(msgpack); +PHP_MINFO_FUNCTION(msgpack); + +PHP_FUNCTION(msgpack_pack); +PHP_FUNCTION(msgpack_unpack); +PHP_FUNCTION(msgpack_unpack_limit); + +PHP_METHOD(msgpack, initialize); +PHP_METHOD(msgpack, execute); +PHP_METHOD(msgpack, execute_limit); +PHP_METHOD(msgpack, finished); +PHP_METHOD(msgpack, data); + +static zend_class_entry *msgpack_ce; + +/* + Declare any global variables you may need between the BEGIN + and END macros here: + +ZEND_BEGIN_MODULE_GLOBALS(msgpack) + long global_value; + char *global_string; +ZEND_END_MODULE_GLOBALS(msgpack) +*/ + +/* In every utility function you add that needs to use variables + in php_msgpack_globals, call TSRMLS_FETCH(); after declaring other + variables used by that function, or better yet, pass in TSRMLS_CC + after the last function argument and declare your utility function + with TSRMLS_DC after the last declared argument. Always refer to + the globals in your function as MSGPACK_G(variable). You are + encouraged to rename these macros something shorter, see + examples in any other php module directory. +*/ + + +#endif /* PHP_MSGPACK_H */ + + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * End: + * vim600: noet sw=4 ts=4 fdm=marker + * vim<600: noet sw=4 ts=4 + */ diff --git a/php/test_normal.php b/php/test_normal.php new file mode 100755 index 0000000..ee348c2 --- /dev/null +++ b/php/test_normal.php @@ -0,0 +1,19 @@ +1), array("takei"=>"hide"), 3); + //$data = array("more"=>10, "test", null); + //$data = array(); + $data = array(0=>1,1=>2,2=>3); + var_dump($data); + + // serialize + $msg = msgpack_pack($data); + + // hexadecimal + $str = unpack('H*', $msg); + var_dump("0x".$str[1]); + + // deserialize + $ret = msgpack_unpack($msg); + var_dump($ret); +?> + diff --git a/php/test_streaming.php b/php/test_streaming.php new file mode 100755 index 0000000..8f89f4c --- /dev/null +++ b/php/test_streaming.php @@ -0,0 +1,34 @@ +initialize(); + $buffer = ""; + $nread = 0; + + foreach($msgs as $msg){ + $buffer = $buffer . $msg; + + while(true){ + $nread = $unpacker->execute($buffer, $nread); + + if($unpacker->finished()){ + $msg = $unpacker->data(); + var_dump($msg); + + $unpacker->initialize(); + $buffer = substr($buffer, $nread); + $nread = 0; + + if(!empty($buffer)){ + continue; + } + } + break; + } + } +?> + diff --git a/php/tests/test_pack.phpt b/php/tests/test_pack.phpt new file mode 100644 index 0000000..595e4d0 --- /dev/null +++ b/php/tests/test_pack.phpt @@ -0,0 +1,169 @@ +--TEST-- +Test msgpack_pack() function : basic functionality +--SKIPIF-- + +--FILE-- + +===DONE=== +--EXPECT-- +*** Testing msgpack_pack() : basic functionality *** +-- Iteration 1 -- +string(4) "0xc0" +-- Iteration 2 -- +string(4) "0xc0" +-- Iteration 3 -- +string(4) "0xc2" +-- Iteration 4 -- +string(4) "0xc2" +-- Iteration 5 -- +string(4) "0xc3" +-- Iteration 6 -- +string(4) "0xc3" +-- Iteration 7 -- +string(4) "0x00" +-- Iteration 8 -- +string(4) "0x7f" +-- Iteration 9 -- +string(6) "0xcc80" +-- Iteration 10 -- +string(8) "0xcd0100" +-- Iteration 11 -- +string(4) "0xff" +-- Iteration 12 -- +string(6) "0xd0df" +-- Iteration 13 -- +string(8) "0xd1ff7f" +-- Iteration 14 -- +string(8) "0x810101" +-- Iteration 15 -- +string(20) "0xcb3ff0000000000000" +-- Iteration 16 -- +string(4) "0x90" +-- Iteration 17 -- +string(34) "0x9f000102030405060708090a0b0c0d0e" +-- Iteration 18 -- +string(40) "0xdc0010000102030405060708090a0b0c0d0e0f" +-- Iteration 19 -- +string(64) "0x8f0100020103020403050406050706080709080a090b0a0c0b0d0c0e0d0f0e" +-- Iteration 20 -- +string(72) "0xde00100100020103020403050406050706080709080a090b0a0c0b0d0c0e0d0f0e100f" +===DONE=== From d639f5747008a6246a81d648c5f3d1cb0e90c162 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 6 Apr 2010 17:46:38 +0900 Subject: [PATCH 0393/1172] ruby: fixes Segmentation fault on MessagePack.unpack(nil) --- ruby/unpack.c | 70 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/ruby/unpack.c b/ruby/unpack.c index df54e75..88a8239 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -220,15 +220,18 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) } -static VALUE MessagePack_Unpacker_execute_impl(VALUE args) +static VALUE MessagePack_Unpacker_execute_do(VALUE argv) { - VALUE self = ((VALUE*)args)[0]; - VALUE data = ((VALUE*)args)[1]; + VALUE* args = (VALUE*)argv; + VALUE self = args[0]; UNPACKER(self, mp); - size_t from = NUM2UINT(((VALUE*)args)[2]); + + VALUE data = args[1]; + + size_t from = (unsigned long)args[2]; char* dptr = RSTRING_PTR(data); - long dlen = FIX2LONG(((VALUE*)args)[3]); + size_t dlen = (unsigned long)args[3]; int ret; if(from >= dlen) { @@ -236,7 +239,7 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE args) } mp->user.source = data; - ret = template_execute(mp, dptr, (size_t)dlen, &from); + ret = template_execute(mp, dptr, dlen, &from); mp->user.source = Qnil; if(ret < 0) { @@ -260,22 +263,32 @@ static VALUE MessagePack_Unpacker_execute_rescue(VALUE nouse) #endif } -static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, - VALUE off, VALUE limit) +static inline VALUE MessagePack_Unpacker_execute_impl(VALUE self, VALUE data, + unsigned long off, unsigned long dlen) { // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ rb_gc_disable(); - VALUE args[4] = {self, data, off, limit}; - VALUE ret = rb_rescue(MessagePack_Unpacker_execute_impl, (VALUE)args, + VALUE args[4] = {self, data, (VALUE)off, (VALUE)dlen}; + VALUE ret = rb_rescue(MessagePack_Unpacker_execute_do, (VALUE)args, MessagePack_Unpacker_execute_rescue, Qnil); rb_gc_enable(); + return ret; } +static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, + VALUE off, VALUE limit) +{ + CHECK_STRING_TYPE(data); + return MessagePack_Unpacker_execute_impl(self, data, + NUM2ULONG(off), NUM2ULONG(limit)); +} + static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) { - return MessagePack_Unpacker_execute_limit(self, data, off, - LONG2FIX(RSTRING_LEN(data))); + CHECK_STRING_TYPE(data); + return MessagePack_Unpacker_execute_impl(self, data, + NUM2ULONG(off), RSTRING_LEN(data)); } static VALUE MessagePack_Unpacker_finished_p(VALUE self) @@ -379,18 +392,20 @@ static VALUE MessagePack_Unpacker_each(VALUE self) } -static VALUE MessagePack_unpack_impl(VALUE args) +static VALUE MessagePack_unpack_do(VALUE argv) { - msgpack_unpack_t* mp = (msgpack_unpack_t*)((VALUE*)args)[0]; - VALUE data = ((VALUE*)args)[1]; + VALUE* args = (VALUE*)argv; + + msgpack_unpack_t* mp = (msgpack_unpack_t*)args[0]; + VALUE data = args[1]; size_t from = 0; char* dptr = RSTRING_PTR(data); - long dlen = FIX2LONG(((VALUE*)args)[2]); + size_t dlen = (unsigned long)args[2]; int ret; mp->user.source = data; - ret = template_execute(mp, dptr, (size_t)dlen, &from); + ret = template_execute(mp, dptr, dlen, &from); mp->user.source = Qnil; if(ret < 0) { @@ -405,7 +420,7 @@ static VALUE MessagePack_unpack_impl(VALUE args) } } -static VALUE MessagePack_unpack_rescue(VALUE args) +static VALUE MessagePack_unpack_rescue(VALUE nouse) { rb_gc_enable(); #ifdef RUBY_VM @@ -415,29 +430,34 @@ static VALUE MessagePack_unpack_rescue(VALUE args) #endif } -static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) +static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned long dlen) { - CHECK_STRING_TYPE(data); - msgpack_unpack_t mp; template_init(&mp); init_stack(&mp); unpack_user u = {0, Qnil, 0, 0, Qnil, Qnil, Qnil}; mp.user = u; + // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ rb_gc_disable(); - VALUE args[3] = {(VALUE)&mp, data, limit}; - VALUE ret = rb_rescue(MessagePack_unpack_impl, (VALUE)args, + VALUE args[3] = {(VALUE)&mp, data, (VALUE)dlen}; + VALUE ret = rb_rescue(MessagePack_unpack_do, (VALUE)args, MessagePack_unpack_rescue, Qnil); rb_gc_enable(); return ret; } +static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) +{ + CHECK_STRING_TYPE(data); + return MessagePack_unpack_impl(self, data, NUM2ULONG(limit)); +} + static VALUE MessagePack_unpack(VALUE self, VALUE data) { - return MessagePack_unpack_limit(self, data, - LONG2FIX(RSTRING_LEN(data))); + CHECK_STRING_TYPE(data); + return MessagePack_unpack_impl(self, data, RSTRING_LEN(data)); } From c6186f2c01e159644b7ed19e865c8ac20d169453 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 6 Apr 2010 17:59:34 +0900 Subject: [PATCH 0394/1172] ruby: version 0.3.7 --- ruby/msgpack.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index 4f7fcec..3a261f2 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.3.6" + s.version = "0.3.7" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" From bd36ac2c0c99fa8e95b50ba12c2cc8ccb8b413f4 Mon Sep 17 00:00:00 2001 From: Masahiro Nakagawa Date: Wed, 7 Apr 2010 19:54:14 +0900 Subject: [PATCH 0395/1172] cpp: fixes argument type of pack_int* methods --- cpp/msgpack/pack.hpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cpp/msgpack/pack.hpp b/cpp/msgpack/pack.hpp index 6a91971..3a68aa6 100644 --- a/cpp/msgpack/pack.hpp +++ b/cpp/msgpack/pack.hpp @@ -1,4 +1,4 @@ -// +1// // MessagePack for C++ serializing routine // // Copyright (C) 2008-2009 FURUHASHI Sadayuki @@ -40,10 +40,10 @@ public: packer& pack_uint16(uint16_t d); packer& pack_uint32(uint32_t d); packer& pack_uint64(uint64_t d); - packer& pack_int8(uint8_t d); - packer& pack_int16(uint16_t d); - packer& pack_int32(uint32_t d); - packer& pack_int64(uint64_t d); + packer& pack_int8(int8_t d); + packer& pack_int16(int16_t d); + packer& pack_int32(int32_t d); + packer& pack_int64(int64_t d); packer& pack_short(short d); packer& pack_int(int d); @@ -166,19 +166,19 @@ inline packer& packer::pack_uint64(uint64_t d) { _pack_uint64(m_stream, d); return *this; } template -inline packer& packer::pack_int8(uint8_t d) +inline packer& packer::pack_int8(int8_t d) { _pack_int8(m_stream, d); return *this; } template -inline packer& packer::pack_int16(uint16_t d) +inline packer& packer::pack_int16(int16_t d) { _pack_int16(m_stream, d); return *this; } template -inline packer& packer::pack_int32(uint32_t d) +inline packer& packer::pack_int32(int32_t d) { _pack_int32(m_stream, d); return *this; } template -inline packer& packer::pack_int64(uint64_t d) +inline packer& packer::pack_int64(int64_t d) { _pack_int64(m_stream, d); return *this;} From e79747a60015e11ab49e10d829f8fdab8ccf8945 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 7 Apr 2010 20:02:00 +0900 Subject: [PATCH 0396/1172] cpp: msgpack/pack.hpp: fixes header --- cpp/msgpack/pack.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/msgpack/pack.hpp b/cpp/msgpack/pack.hpp index 3a68aa6..ee90690 100644 --- a/cpp/msgpack/pack.hpp +++ b/cpp/msgpack/pack.hpp @@ -1,7 +1,7 @@ -1// +// // MessagePack for C++ serializing routine // -// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// Copyright (C) 2008-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From 1f18af43955b22eed97b6fcb7b53c0389e4bc0f9 Mon Sep 17 00:00:00 2001 From: Masahiro Nakagawa Date: Thu, 8 Apr 2010 23:01:19 +0900 Subject: [PATCH 0397/1172] c: fixes comment in pack_template.h --- msgpack/pack_template.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 493634d..f8f522f 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -113,17 +113,17 @@ do { \ } \ } else { \ if(d < (1ULL<<16)) { \ - /* signed 16 */ \ + /* unsigned 16 */ \ unsigned char buf[3]; \ buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ - /* signed 32 */ \ + /* unsigned 32 */ \ unsigned char buf[5]; \ buf[0] = 0xce; _msgpack_store32(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ - /* signed 64 */ \ + /* unsigned 64 */ \ unsigned char buf[9]; \ buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 9); \ From 87835a4e60e8f7ba5c2d4283ed44f57b725d9628 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 14 Apr 2010 21:08:06 +0900 Subject: [PATCH 0398/1172] ruby: remove init_stack, adopt rb_gc_mark_maybe --- ruby/makegem.sh | 4 ++++ ruby/msgpack_test.rb | 15 +++++++++++++++ ruby/unpack.c | 14 +------------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/ruby/makegem.sh b/ruby/makegem.sh index c78769b..8737d29 100755 --- a/ruby/makegem.sh +++ b/ruby/makegem.sh @@ -19,6 +19,10 @@ cat msgpack_test.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FI gem build msgpack.gemspec +if [ $? -eq 0 ]; then + rm -rf ext msgpack AUTHORS ChangeLog test/msgpack_test.rb +fi + # gem install gem-compile # on msys # gem compile msgpack-$version.gem # on msys # gem compile msgpack-$version.gem -p mswin32 # on msys diff --git a/ruby/msgpack_test.rb b/ruby/msgpack_test.rb index 4fbcea3..37db6a0 100644 --- a/ruby/msgpack_test.rb +++ b/ruby/msgpack_test.rb @@ -202,6 +202,21 @@ class MessagePackTestFormat < Test::Unit::TestCase # #check_map 5, (1<<32)-1 # memory error # end + it "gc mark" do + obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"] + pac = MessagePack::Unpacker.new + parsed = 0 + obj.to_msgpack.split(//).each do |b| + pac.feed(b) + pac.each {|o| + assert_equal(obj, o) + parsed += 1 + } + GC.start + end + assert_equal(parsed, 1) + end + private def check(len, obj) v = obj.to_msgpack diff --git a/ruby/unpack.c b/ruby/unpack.c index 88a8239..5f5b64b 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -141,15 +141,6 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha static VALUE cUnpacker; static VALUE eUnpackError; -// FIXME slow operation -static void init_stack(msgpack_unpack_t* mp) -{ - size_t i; - for(i=0; i < MSGPACK_MAX_STACK_SIZE; ++i) { - mp->stack[i].map_key = Qnil; /* GC */ - } -} - static void MessagePack_Unpacker_free(void* data) { if(data) { free(data); } @@ -163,7 +154,7 @@ static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp) rb_gc_mark(mp->user.streambuf); for(i=0; i < mp->top; ++i) { rb_gc_mark(mp->stack[i].obj); - rb_gc_mark(mp->stack[i].map_key); /* maybe map_key is not initialized */ + rb_gc_mark_maybe(mp->stack[i].map_key); } } @@ -180,7 +171,6 @@ static VALUE MessagePack_Unpacker_reset(VALUE self) { UNPACKER(self, mp); template_init(mp); - init_stack(mp); mp->user.finished = 0; return self; } @@ -381,7 +371,6 @@ static VALUE MessagePack_Unpacker_each(VALUE self) } else if(ret > 0) { VALUE data = template_data(mp); template_init(mp); - init_stack(mp); rb_yield(data); } else { goto do_fill; @@ -434,7 +423,6 @@ static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned lon { msgpack_unpack_t mp; template_init(&mp); - init_stack(&mp); unpack_user u = {0, Qnil, 0, 0, Qnil, Qnil, Qnil}; mp.user = u; From a6ec726ed7ca72662f0abbc8e1d8e361201483ae Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 14 Apr 2010 21:11:31 +0900 Subject: [PATCH 0399/1172] malloc/realloc the stack when its length becomes > MSGPACK_EMBED_STACK_SIZE --- c/unpack.c | 22 +++++++++++------ cpp/msgpack/object.hpp | 52 ++++++++++++++++++++++++++++++++++++--- msgpack/unpack_define.h | 5 ++-- msgpack/unpack_template.h | 33 ++++++++++++++++++++++--- perl/unpack.c | 8 +++++- 5 files changed, 103 insertions(+), 17 deletions(-) diff --git a/c/unpack.c b/c/unpack.c index 8be0b92..4334974 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -44,6 +44,7 @@ struct template_context; typedef struct template_context template_context; static void template_init(template_context* ctx); +static void template_destroy(template_context* ctx); static msgpack_object template_data(template_context* ctx); @@ -215,6 +216,7 @@ bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) void msgpack_unpacker_destroy(msgpack_unpacker* mpac) { msgpack_zone_free(mpac->z); + template_destroy(mpac->ctx); free(mpac->ctx); decl_count(mpac->buffer); } @@ -368,6 +370,7 @@ msgpack_unpack_return msgpack_unpack(const char* data, size_t len, size_t* off, msgpack_zone* z, msgpack_object* result) { + msgpack_unpack_return ret = MSGPACK_UNPACK_SUCCESS; template_context ctx; template_init(&ctx); @@ -377,23 +380,28 @@ msgpack_unpack(const char* data, size_t len, size_t* off, size_t noff = 0; if(off != NULL) { noff = *off; } - int ret = template_execute(&ctx, data, len, &noff); - if(ret < 0) { - return MSGPACK_UNPACK_PARSE_ERROR; + int e = template_execute(&ctx, data, len, &noff); + if(e < 0) { + ret = MSGPACK_UNPACK_PARSE_ERROR; + goto out; } if(off != NULL) { *off = noff; } - if(ret == 0) { - return MSGPACK_UNPACK_CONTINUE; + if(e == 0) { + ret = MSGPACK_UNPACK_CONTINUE; + goto out; } *result = template_data(&ctx); if(noff < len) { - return MSGPACK_UNPACK_EXTRA_BYTES; + ret = MSGPACK_UNPACK_EXTRA_BYTES; + goto out; } - return MSGPACK_UNPACK_SUCCESS; +out: + template_destroy(&ctx); + return ret; } diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index ed2e290..3b42a8e 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -87,9 +87,15 @@ struct object { template void convert(T* v) const; - object(); - object(msgpack_object obj); operator msgpack_object(); + object(msgpack_object obj); + + object(); + object(bool v); + object(uint64_t v); + object(int64_t v); + object(double v); + object(const char* ptr, size_t size); private: struct implicit_type; @@ -184,7 +190,46 @@ inline bool operator!=(const object x, const object y) { return !(x == y); } -inline object::object() { } +inline object::object() +{ + type = type::NIL; +} + +inline object::object(bool v) +{ + type = type::BOOLEAN; + via.boolean = v; +} + +inline object::object(uint64_t v) +{ + type = type::POSITIVE_INTEGER; + via.u64 = v; +} + +inline object::object(int64_t v) +{ + if(v >= 0) { + type = type::POSITIVE_INTEGER; + via.u64 = v; + } else { + type = type::NEGATIVE_INTEGER; + via.i64 = v; + } +} + +inline object::object(double v) +{ + type = type::DOUBLE; + via.dec = v; +} + +inline object::object(const char* ptr, size_t size) +{ + type = type::RAW; + via.raw.size = size; + via.raw.ptr = ptr; +} inline object::object(msgpack_object obj) { @@ -200,7 +245,6 @@ inline object::operator msgpack_object() return obj; } - inline object::implicit_type object::convert() const { return implicit_type(*this); diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index e61f73d..cb02a3b 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -19,6 +19,7 @@ #define MSGPACK_UNPACK_DEFINE_H__ #include "msgpack/sysdep.h" +#include #include #include #include @@ -28,8 +29,8 @@ extern "C" { #endif -#ifndef MSGPACK_MAX_STACK_SIZE -#define MSGPACK_MAX_STACK_SIZE 16 +#ifndef MSGPACK_EMBED_STACK_SIZE +#define MSGPACK_EMBED_STACK_SIZE 1 #endif diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 3f871cd..72f8e59 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -58,7 +58,9 @@ msgpack_unpack_struct_decl(_context) { unsigned int cs; unsigned int trail; unsigned int top; - msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE]; + msgpack_unpack_struct(_stack)* stack; + unsigned int stack_size; + msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE]; }; @@ -67,9 +69,18 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; + ctx->stack = ctx->embed_stack; + ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); } +msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx) +{ + if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { + free(ctx->stack); + } +} + msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) { return (ctx)->stack[0].obj; @@ -119,12 +130,28 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c #define start_container(func, count_, ct_) \ if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ if((count_) == 0) { obj = stack[top].obj; goto _push; } \ - if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ stack[top].ct = ct_; \ stack[top].count = count_; \ + ++top; \ /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ /*printf("stack push %d\n", top);*/ \ - ++top; \ + if(top >= ctx->stack_size) { \ + if(ctx->stack_size == MSGPACK_EMBED_STACK_SIZE) { \ + size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \ + size_t nsize = csize * 2; \ + msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \ + if(tmp == NULL) { goto _failed; } \ + memcpy(tmp, ctx->stack, csize); \ + ctx->stack = tmp; \ + ctx->stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ + } else { \ + size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \ + msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \ + if(tmp == NULL) { goto _failed; } \ + ctx->stack = tmp; \ + ctx->stack_size *= 2; \ + } \ + } \ goto _header_again #define NEXT_CS(p) \ diff --git a/perl/unpack.c b/perl/unpack.c index 7a4f422..c520e02 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -53,6 +53,7 @@ struct template_context; typedef struct template_context msgpack_unpack_t; static void template_init(msgpack_unpack_t* u); +static void template_destroy(msgpack_unpack_t* u); static SV* template_data(msgpack_unpack_t* u); @@ -132,6 +133,7 @@ SV* _msgpack_unpack(SV* data, int limit) { size_t from = 0; STRLEN dlen; const char * dptr = SvPV_const(data, dlen); + SV* obj; template_init(&mp); mp.user = u; @@ -140,6 +142,9 @@ SV* _msgpack_unpack(SV* data, int limit) { ret = template_execute(&mp, dptr, (size_t)dlen, &from); mp.user.source = &PL_sv_undef; + obj = template_data(&mp); + template_destroy(&mp); + if(ret < 0) { Perl_croak(aTHX_ "parse error."); } else if(ret == 0) { @@ -148,7 +153,7 @@ SV* _msgpack_unpack(SV* data, int limit) { if(from < dlen) { Perl_croak(aTHX_ "extra bytes."); } - return template_data(&mp); + return obj; } } @@ -311,6 +316,7 @@ XS(xs_unpacker_destroy) { } UNPACKER(ST(0), mp); + template_destroy(mp); Safefree(mp); XSRETURN(0); From a55affe4d5bce92ee692fd80b10365b6ec49804c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 17 Apr 2010 16:16:56 +0900 Subject: [PATCH 0400/1172] ruby: add Symbol#to_msgpack --- ruby/pack.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ruby/pack.c b/ruby/pack.c index 1d0068c..387bab6 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -112,6 +112,16 @@ static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } +static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + const char* name = rb_id2name(SYM2ID(self)); + size_t len = strlen(name); + msgpack_pack_raw(out, len); + msgpack_pack_raw_body(out, name, len); + return out; +} + static VALUE MessagePack_Array_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -172,6 +182,7 @@ void Init_msgpack_pack(VALUE mMessagePack) rb_define_method_id(rb_cString, s_to_msgpack, MessagePack_String_to_msgpack, -1); rb_define_method_id(rb_cArray, s_to_msgpack, MessagePack_Array_to_msgpack, -1); rb_define_method_id(rb_cHash, s_to_msgpack, MessagePack_Hash_to_msgpack, -1); + rb_define_method_id(rb_cSymbol, s_to_msgpack, MessagePack_Symbol_to_msgpack, -1); rb_define_module_function(mMessagePack, "pack", MessagePack_pack, -1); } From 228f742b2f4bfba8bc97f16b3fb85ca616b07768 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 17 Apr 2010 20:02:47 +0900 Subject: [PATCH 0401/1172] ruby: set encoding to 'ASCII-8BIT' before deserializing on ruby-1.9 --- ruby/unpack.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/ruby/unpack.c b/ruby/unpack.c index 5f5b64b..11572c3 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -16,11 +16,17 @@ * limitations under the License. */ #include "ruby.h" + #include "msgpack/unpack_define.h" static ID s_sysread; static ID s_readpartial; +#ifdef HAVE_RUBY_ENCODING_H +#include "ruby/encoding.h" +int s_ascii_8bit; +#endif + typedef struct { int finished; VALUE source; @@ -426,6 +432,12 @@ static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned lon unpack_user u = {0, Qnil, 0, 0, Qnil, Qnil, Qnil}; mp.user = u; +#ifdef HAVE_RUBY_ENCODING_H + // FIXME encodingã‚’ASCII-8BITã«ã™ã‚‹ + int enc_orig = rb_enc_get_index(data); + rb_enc_set_index(data, s_ascii_8bit); +#endif + // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ rb_gc_disable(); VALUE args[3] = {(VALUE)&mp, data, (VALUE)dlen}; @@ -433,6 +445,10 @@ static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned lon MessagePack_unpack_rescue, Qnil); rb_gc_enable(); +#ifdef HAVE_RUBY_ENCODING_H + rb_enc_set_index(data, enc_orig); +#endif + return ret; } @@ -453,6 +469,11 @@ void Init_msgpack_unpack(VALUE mMessagePack) { s_sysread = rb_intern("sysread"); s_readpartial = rb_intern("readpartial"); + +#ifdef HAVE_RUBY_ENCODING_H + s_ascii_8bit = rb_enc_find_index("ASCII-8BIT"); +#endif + eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); From b4b1f0a2c961529cdabd705b2581350323952f74 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sat, 17 Apr 2010 20:16:48 +0900 Subject: [PATCH 0402/1172] introduce JUnit framework for testing. --- java/build.xml | 207 +++++++++++++++++++++++--- java/ivy.xml | 19 +++ java/test/org/msgpack/TestSample.java | 11 ++ 3 files changed, 220 insertions(+), 17 deletions(-) create mode 100644 java/ivy.xml create mode 100644 java/test/org/msgpack/TestSample.java diff --git a/java/build.xml b/java/build.xml index b98f7c3..35ee2fe 100644 --- a/java/build.xml +++ b/java/build.xml @@ -1,18 +1,191 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Tests Failed! + + + + + + + + + + diff --git a/java/ivy.xml b/java/ivy.xml new file mode 100644 index 0000000..6d98756 --- /dev/null +++ b/java/ivy.xml @@ -0,0 +1,19 @@ + + + + + MessagePack + + + + + + + + + + + + diff --git a/java/test/org/msgpack/TestSample.java b/java/test/org/msgpack/TestSample.java new file mode 100644 index 0000000..d0e3bae --- /dev/null +++ b/java/test/org/msgpack/TestSample.java @@ -0,0 +1,11 @@ +package org.msgpack; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +public class TestSample { + @Test + public void testNull() throws Exception { + assertEquals("aiueo", 0, 0); + } +}; From 08b716c96d02e281cc096783cbe64932f70919ef Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sat, 17 Apr 2010 21:16:32 +0900 Subject: [PATCH 0403/1172] java: add TestPackUnpack for int --- java/build.xml | 8 +++- java/test/org/msgpack/TestPackUnpack.java | 52 +++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 java/test/org/msgpack/TestPackUnpack.java diff --git a/java/build.xml b/java/build.xml index 35ee2fe..baecd01 100644 --- a/java/build.xml +++ b/java/build.xml @@ -14,6 +14,7 @@ + @@ -64,6 +65,9 @@ + + + @@ -146,7 +150,7 @@ - + @@ -178,7 +182,7 @@ Tests Failed! - + diff --git a/java/test/org/msgpack/TestPackUnpack.java b/java/test/org/msgpack/TestPackUnpack.java new file mode 100644 index 0000000..f8eeae7 --- /dev/null +++ b/java/test/org/msgpack/TestPackUnpack.java @@ -0,0 +1,52 @@ +package org.msgpack; + +import org.msgpack.*; +import java.io.*; +import java.util.*; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class TestPackUnpack { + public Object unpackOne(ByteArrayOutputStream out) { + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker upk = new Unpacker(in); + Iterator it = upk.iterator(); + assertEquals(true, it.hasNext()); + Object obj = it.next(); + assertEquals(false, it.hasNext()); + return obj; + } + + @Test + public void testInt() throws Exception { + testInt(0); + testInt(-1); + testInt(1); + testInt(Integer.MIN_VALUE); + testInt(Integer.MAX_VALUE); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testInt(rand.nextInt()); + } + public void testInt(int val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Packer pk = new Packer(out); + pk.pack(val); + Object obj = unpackOne(out); + int val2 = -1; + if (obj instanceof Byte) + val2 = ((Byte)obj).intValue(); + else if (obj instanceof Integer) + val2 = ((Integer)obj).intValue(); + else if (obj instanceof Short) + val2 = ((Short)obj).intValue(); + else if (obj instanceof Long) + val2 = ((Long)obj).intValue(); + else { + System.out.println("obj = " + obj.getClass()); + assertTrue(false); + } + assertEquals(val, val2); + } +}; From 2807504a814fd54f91b32e58476a80b5ca40c2fc Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sat, 17 Apr 2010 22:10:41 +0900 Subject: [PATCH 0404/1172] java: add tests for float, double, nil, boolean, string --- java/test/org/msgpack/TestPackUnpack.java | 140 ++++++++++++++++++++-- 1 file changed, 130 insertions(+), 10 deletions(-) diff --git a/java/test/org/msgpack/TestPackUnpack.java b/java/test/org/msgpack/TestPackUnpack.java index f8eeae7..1a02cc5 100644 --- a/java/test/org/msgpack/TestPackUnpack.java +++ b/java/test/org/msgpack/TestPackUnpack.java @@ -8,7 +8,7 @@ import org.junit.Test; import static org.junit.Assert.*; public class TestPackUnpack { - public Object unpackOne(ByteArrayOutputStream out) { + protected Object unpackOne(ByteArrayOutputStream out) { ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); Unpacker upk = new Unpacker(in); Iterator it = upk.iterator(); @@ -31,22 +31,142 @@ public class TestPackUnpack { } public void testInt(int val) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); - Packer pk = new Packer(out); - pk.pack(val); + new Packer(out).pack(val); Object obj = unpackOne(out); - int val2 = -1; if (obj instanceof Byte) - val2 = ((Byte)obj).intValue(); + assertEquals(val, ((Byte)obj).intValue()); else if (obj instanceof Integer) - val2 = ((Integer)obj).intValue(); + assertEquals(val, ((Integer)obj).intValue()); else if (obj instanceof Short) - val2 = ((Short)obj).intValue(); + assertEquals(val, ((Short)obj).intValue()); else if (obj instanceof Long) - val2 = ((Long)obj).intValue(); + assertEquals(val, ((Long)obj).intValue()); else { - System.out.println("obj = " + obj.getClass()); + System.out.println("Got unexpected class: " + obj.getClass()); + assertTrue(false); + } + } + + @Test + public void testFloat() throws Exception { + testFloat((float)0.0); + testFloat((float)-0.0); + testFloat((float)1.0); + testFloat((float)-1.0); + testFloat((float)Float.MAX_VALUE); + testFloat((float)Float.MIN_VALUE); + testFloat((float)Float.NaN); + testFloat((float)Float.NEGATIVE_INFINITY); + testFloat((float)Float.POSITIVE_INFINITY); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testFloat(rand.nextFloat()); + } + public void testFloat(float val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + Object obj = unpackOne(out); + if (obj instanceof Float) + assertEquals(val, ((Float)obj).floatValue(), 10e-10); + else { + System.out.println("Got unexpected class: " + obj.getClass()); + assertTrue(false); + } + } + + @Test + public void testDouble() throws Exception { + testDouble((double)0.0); + testDouble((double)-0.0); + testDouble((double)1.0); + testDouble((double)-1.0); + testDouble((double)Double.MAX_VALUE); + testDouble((double)Double.MIN_VALUE); + testDouble((double)Double.NaN); + testDouble((double)Double.NEGATIVE_INFINITY); + testDouble((double)Double.POSITIVE_INFINITY); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testDouble(rand.nextDouble()); + } + public void testDouble(double val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + Object obj = unpackOne(out); + if (obj instanceof Double) + assertEquals(val, ((Double)obj).doubleValue(), 10e-10); + else { + System.out.println("Got unexpected class: " + obj.getClass()); + assertTrue(false); + } + } + + @Test + public void testNil() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).packNil(); + Object obj = unpackOne(out); + assertEquals(null, obj); + } + + @Test + public void testBoolean() throws Exception { + testBoolean(false); + testBoolean(true); + } + public void testBoolean(boolean val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + Object obj = unpackOne(out); + if (obj instanceof Boolean) + assertEquals(val, ((Boolean)obj).booleanValue()); + else { + System.out.println("Got unexpected class: " + obj.getClass()); + assertTrue(false); + } + } + + @Test + public void testString() throws Exception { + testString(""); + testString("a"); + testString("ab"); + testString("abc"); + // small size string + for (int i = 0; i < 100; i++) { + StringBuilder sb = new StringBuilder(); + int len = (int)Math.random() % 31 + 1; + for (int j = 0; j < len; j++) + sb.append('a' + ((int)Math.random()) & 26); + testString(sb.toString()); + } + // medium size string + for (int i = 0; i < 100; i++) { + StringBuilder sb = new StringBuilder(); + int len = (int)Math.random() % 100 + (1 << 15); + for (int j = 0; j < len; j++) + sb.append('a' + ((int)Math.random()) & 26); + testString(sb.toString()); + } + // large size string + for (int i = 0; i < 10; i++) { + StringBuilder sb = new StringBuilder(); + int len = (int)Math.random() % 100 + (1 << 31); + for (int j = 0; j < len; j++) + sb.append('a' + ((int)Math.random()) & 26); + testString(sb.toString()); + } + } + public void testString(String val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + Object obj = unpackOne(out); + if (obj instanceof byte[]) + assertEquals(val, new String((byte[])obj)); + else { + System.out.println("obj=" + obj); + System.out.println("Got unexpected class: " + obj.getClass()); assertTrue(false); } - assertEquals(val, val2); } }; From c3f43fb0cf14f596a9ebb1c30b0417b3a8ba9958 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 17 Apr 2010 22:43:11 +0900 Subject: [PATCH 0405/1172] template_execute: fixes embed stack --- msgpack/unpack_define.h | 2 +- msgpack/unpack_template.h | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index cb02a3b..71412ee 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -30,7 +30,7 @@ extern "C" { #ifndef MSGPACK_EMBED_STACK_SIZE -#define MSGPACK_EMBED_STACK_SIZE 1 +#define MSGPACK_EMBED_STACK_SIZE 16 #endif diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 72f8e59..4b8cd14 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -99,6 +99,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c unsigned int cs = ctx->cs; unsigned int top = ctx->top; msgpack_unpack_struct(_stack)* stack = ctx->stack; + unsigned int stack_size = ctx->stack_size; msgpack_unpack_user* user = &ctx->user; msgpack_unpack_object obj; @@ -135,21 +136,21 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ++top; \ /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ /*printf("stack push %d\n", top);*/ \ - if(top >= ctx->stack_size) { \ - if(ctx->stack_size == MSGPACK_EMBED_STACK_SIZE) { \ + if(top >= stack_size) { \ + if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \ size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \ size_t nsize = csize * 2; \ msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \ if(tmp == NULL) { goto _failed; } \ memcpy(tmp, ctx->stack, csize); \ - ctx->stack = tmp; \ - ctx->stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ + ctx->stack = stack = tmp; \ + ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ } else { \ size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \ msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \ if(tmp == NULL) { goto _failed; } \ - ctx->stack = tmp; \ - ctx->stack_size *= 2; \ + ctx->stack = stack = tmp; \ + ctx->stack_size = stack_size = stack_size * 2; \ } \ } \ goto _header_again From a65438c6feaf82aa137b84432bd66722e944e5bd Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sat, 17 Apr 2010 22:53:56 +0900 Subject: [PATCH 0406/1172] java: skip building jar file for the faster testing --- java/build.xml | 27 +++++++-------------------- java/ivy.xml | 2 +- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/java/build.xml b/java/build.xml index baecd01..52c5218 100644 --- a/java/build.xml +++ b/java/build.xml @@ -5,25 +5,18 @@ - - - - - + - - - @@ -65,13 +58,9 @@ - - - - @@ -80,10 +69,10 @@ + - @@ -145,12 +134,12 @@ - - - + + + - + @@ -166,8 +155,6 @@ errorProperty="tests.failed" failureProperty="tests.failed"> - - @@ -182,7 +169,7 @@ Tests Failed! - + diff --git a/java/ivy.xml b/java/ivy.xml index 6d98756..a694214 100644 --- a/java/ivy.xml +++ b/java/ivy.xml @@ -1,7 +1,7 @@ - MessagePack From abeed3be84b7b12b71598c3535c4c9a6e70c534f Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sat, 17 Apr 2010 23:25:42 +0900 Subject: [PATCH 0407/1172] java: add tests for array and map --- java/test/org/msgpack/TestPackUnpack.java | 69 +++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/java/test/org/msgpack/TestPackUnpack.java b/java/test/org/msgpack/TestPackUnpack.java index 1a02cc5..6877853 100644 --- a/java/test/org/msgpack/TestPackUnpack.java +++ b/java/test/org/msgpack/TestPackUnpack.java @@ -9,8 +9,13 @@ import static org.junit.Assert.*; public class TestPackUnpack { protected Object unpackOne(ByteArrayOutputStream out) { + return unpackOne(out, null); + } + protected Object unpackOne(ByteArrayOutputStream out, Schema schema) { ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); Unpacker upk = new Unpacker(in); + if (schema != null) + upk = upk.useSchema(schema); Iterator it = upk.iterator(); assertEquals(true, it.hasNext()); Object obj = it.next(); @@ -169,4 +174,68 @@ public class TestPackUnpack { assertTrue(false); } } + + @Test + public void testArray() throws Exception { + for (int i = 0; i < 1000; i++) { + Schema schema = Schema.parse("(array int)"); + List l = new ArrayList(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + l.add(j); + testArray(l, schema); + } + for (int i = 0; i < 1000; i++) { + Schema schema = Schema.parse("(array string)"); + List l = new ArrayList(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + l.add(Integer.toString(j)); + testArray(l, schema); + } + } + public void testArray(List val, Schema schema) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + Object obj = unpackOne(out, schema); + if (obj instanceof List) + assertTrue(val.equals(obj)); + else { + System.out.println("obj=" + obj); + System.out.println("Got unexpected class: " + obj.getClass()); + assertTrue(false); + } + } + + @Test + public void testMap() throws Exception { + for (int i = 0; i < 1000; i++) { + Schema schema = Schema.parse("(map int int)"); + Map m = new HashMap(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + m.put(j, j); + testMap(m, schema); + } + for (int i = 0; i < 1000; i++) { + Schema schema = Schema.parse("(map string int)"); + Map m = new HashMap(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + m.put(Integer.toString(j), j); + testMap(m, schema); + } + } + public void testMap(Map val, Schema schema) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + Object obj = unpackOne(out, schema); + if (obj instanceof Map) + assertTrue(val.equals(obj)); + else { + System.out.println("obj=" + obj); + System.out.println("Got unexpected class: " + obj.getClass()); + assertTrue(false); + } + } }; From ab8e0c9e31585b24f1d4cbfb1e73b0ce249e408a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 18 Apr 2010 00:08:03 +0900 Subject: [PATCH 0408/1172] c,cpp: reforms source tree --- Makefile.am | 18 -- NEWS | 0 README | 70 ---- README.md | 37 +++ c/Makefile.am | 29 -- c/bench.c | 323 ------------------- c/bench.mk | 9 - AUTHORS => cpp/AUTHORS | 0 COPYING => cpp/COPYING | 0 ChangeLog => cpp/ChangeLog | 0 LICENSE => cpp/LICENSE | 0 cpp/Makefile.am | 56 +++- NOTICE => cpp/NOTICE | 0 cpp/bench.cpp | 188 ----------- cpp/bench.mk | 9 - bootstrap => cpp/bootstrap | 11 +- configure.in => cpp/configure.in | 36 +-- {c => cpp}/msgpack.h | 0 {c => cpp}/msgpack/object.h | 0 {c => cpp}/msgpack/pack.h | 0 {c => cpp}/msgpack/sbuffer.h | 0 {c => cpp}/msgpack/unpack.h | 0 {c => cpp}/msgpack/vrefbuffer.h | 0 {c => cpp}/msgpack/zbuffer.h | 0 {c => cpp}/msgpack/zone.h | 0 cpp/{test.cpp => msgpack_test.cpp} | 0 cpp/msgpack_vc8.postbuild.bat | 49 +++ msgpack_vc8.sln => cpp/msgpack_vc8.sln | 0 msgpack_vc8.vcproj => cpp/msgpack_vc8.vcproj | 0 c/test.cpp => cpp/msgpackc_test.cpp | 0 {c => cpp}/object.c | 0 cpp/{preprocess.sh => preprocess} | 4 + {c => cpp}/unpack.c | 0 {c => cpp}/vrefbuffer.c | 0 {c => cpp}/zone.c | 0 msgpack_vc8.postbuild.bat | 41 --- 36 files changed, 159 insertions(+), 721 deletions(-) delete mode 100644 Makefile.am delete mode 100644 NEWS delete mode 100644 README create mode 100644 README.md delete mode 100644 c/Makefile.am delete mode 100644 c/bench.c delete mode 100644 c/bench.mk rename AUTHORS => cpp/AUTHORS (100%) rename COPYING => cpp/COPYING (100%) rename ChangeLog => cpp/ChangeLog (100%) rename LICENSE => cpp/LICENSE (100%) rename NOTICE => cpp/NOTICE (100%) delete mode 100644 cpp/bench.cpp delete mode 100644 cpp/bench.mk rename bootstrap => cpp/bootstrap (93%) rename configure.in => cpp/configure.in (61%) rename {c => cpp}/msgpack.h (100%) rename {c => cpp}/msgpack/object.h (100%) rename {c => cpp}/msgpack/pack.h (100%) rename {c => cpp}/msgpack/sbuffer.h (100%) rename {c => cpp}/msgpack/unpack.h (100%) rename {c => cpp}/msgpack/vrefbuffer.h (100%) rename {c => cpp}/msgpack/zbuffer.h (100%) rename {c => cpp}/msgpack/zone.h (100%) rename cpp/{test.cpp => msgpack_test.cpp} (100%) create mode 100644 cpp/msgpack_vc8.postbuild.bat rename msgpack_vc8.sln => cpp/msgpack_vc8.sln (100%) rename msgpack_vc8.vcproj => cpp/msgpack_vc8.vcproj (100%) mode change 100755 => 100644 rename c/test.cpp => cpp/msgpackc_test.cpp (100%) rename {c => cpp}/object.c (100%) rename cpp/{preprocess.sh => preprocess} (62%) rename {c => cpp}/unpack.c (100%) rename {c => cpp}/vrefbuffer.c (100%) rename {c => cpp}/zone.c (100%) delete mode 100644 msgpack_vc8.postbuild.bat diff --git a/Makefile.am b/Makefile.am deleted file mode 100644 index 04f62d2..0000000 --- a/Makefile.am +++ /dev/null @@ -1,18 +0,0 @@ -if ENABLE_CXX -SUBDIRS = c cpp -else -SUBDIRS = c -endif - -nobase_include_HEADERS = \ - msgpack/pack_define.h \ - msgpack/pack_template.h \ - msgpack/unpack_define.h \ - msgpack/unpack_template.h \ - msgpack/sysdep.h - -EXTRA_DIST = \ - msgpack_vc8.vcproj \ - msgpack_vc8.sln \ - msgpack_vc8.postbuild.bat - diff --git a/NEWS b/NEWS deleted file mode 100644 index e69de29..0000000 diff --git a/README b/README deleted file mode 100644 index 6ca10d6..0000000 --- a/README +++ /dev/null @@ -1,70 +0,0 @@ -MessagePack ------------ -Binary-based efficient data interchange format. - - -*Requirements - - MessagePack is only tested on Linux and Mac OS X, but it may run on other - UNIX-like platforms. - - gcc >= 4.1 is required to build. - - -*Installation - - Simply run ./configure && make && make install to install C and C++ binding. - - $ ./configure - $ make - $ sudo make install - - To install Ruby binding, run ./makegem.sh script on ruby/ directory and install - generated gem package. - - $ cd ruby - $ ./makegem.sh - $ gem install msgpack-*.gem - - -*Usage - - C++: - include msgpack.hpp header and link libmsgpack library. - see example/simple.cc for example. - - g++ simple.cc -lmsgpack - g++ stream.cc -lmsgpack -lpthread - - - C: - include msgpack.h header and link libmsgpackc library. - see example/simple.c for example. - - gcc simple.c -lmsgpackc - - - Ruby: - require msgpack library. - see example/simple.rb for example. - - ruby -rubygems simple.rb - - - API Document is available at http://msgpack.sourceforge.jp/. - - -Copyright (C) 2008-2010 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/README.md b/README.md new file mode 100644 index 0000000..066c45e --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +MessagePack +=========== +Extremely efficient object serialization library. It's like JSON, but very fast and small. + + +## What's MessagePack? + +MessagePack is a binary-based efficient object serialization library. It enables to exchange structured objects between many languages like JSON. But unlike JSON, it is very fast and small. + +Typical small integer (like flags or error code) is saved only in 1 byte, and typical short string only needs 1 byte except the length of the string itself. \[1,2,3\] (3 elements array) is serialized in 4 bytes using MessagePack as follows: + + require 'msgpack' + msg = [1,2,3].to_msgpack #=> "\x93\x01\x02\x03" + MessagePack.unpack(msg) #=> [1,2,3] + + +## Performance + +![Serialization + Deserialization Speed Test](http://msgpack.sourceforge.net/index/speedtest.png) + +In this test, it measured the elapsed time of serializing and deserializing 200,000 target objects. The target object consists of the three integers and 512 bytes string. +The source code of this test is available from [frsyuki' serializer-speed-test repository.](http://github.com/frsyuki/serializer-speed-test) + + +## Getting Started + +Usage and other documents about implementations in each language are found at [the web site.](http://msgpack.sourceforge.net/) + + +## Learn More + + - [Project Web Site](http://msgpack.sourceforge.net/) + - [MessagePack format specification](http://msgpack.sourceforge.net/spec) + - [Repository at github](http://github.com/msgpack/msgpack) + - [Wiki](http://msgpack.sourceforge.net/start) + - [MessagePack-RPC](http://github.com/msgpack/msgpack-rpc) + diff --git a/c/Makefile.am b/c/Makefile.am deleted file mode 100644 index bbe547c..0000000 --- a/c/Makefile.am +++ /dev/null @@ -1,29 +0,0 @@ -lib_LTLIBRARIES = libmsgpackc.la - -libmsgpackc_la_SOURCES = \ - unpack.c \ - object.c \ - vrefbuffer.c \ - zone.c - -nobase_include_HEADERS = \ - msgpack.h \ - msgpack/sbuffer.h \ - msgpack/vrefbuffer.h \ - msgpack/zbuffer.h \ - msgpack/pack.h \ - msgpack/unpack.h \ - msgpack/object.h \ - msgpack/zone.h - -# -version-info CURRENT:REVISION:AGE -libmsgpackc_la_LDFLAGS = -version-info 2:0:0 - -check_PROGRAMS = \ - msgpackc_test - -msgpackc_test_SOURCES = test.cpp -msgpackc_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -msgpackc_test_LDADD = libmsgpackc.la -lgtest_main - -TESTS = $(check_PROGRAMS) diff --git a/c/bench.c b/c/bench.c deleted file mode 100644 index d72a10d..0000000 --- a/c/bench.c +++ /dev/null @@ -1,323 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include -#include - - -static struct timeval g_timer; - -void reset_timer() -{ - gettimeofday(&g_timer, NULL); -} - -void show_timer(size_t bufsz) -{ - struct timeval endtime; - gettimeofday(&endtime, NULL); - double sec = (endtime.tv_sec - g_timer.tv_sec) - + (double)(endtime.tv_usec - g_timer.tv_usec) / 1000 / 1000; - printf("%f sec\n", sec); - printf("%f MB\n", ((double)bufsz)/1024/1024); - printf("%f Mbps\n", ((double)bufsz)*8/sec/1000/1000); -} - - -static int reformat_null(void * ctx) { return 1; } -static int reformat_boolean(void * ctx, int boolean) { return 1; } -static int reformat_number(void * ctx, const char * s, unsigned int l) { return 1; } -static int reformat_string(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } -static int reformat_map_key(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } -static int reformat_start_map(void * ctx) { return 1; } -static int reformat_end_map(void * ctx) { return 1; } -static int reformat_start_array(void * ctx) { return 1; } -static int reformat_end_array(void * ctx) { return 1; } - - -static void* unpack_uint8(void* data, uint8_t d) { return NULL; } -static void* unpack_uint16(void* data, uint16_t d) { return NULL; } -static void* unpack_uint32(void* data, uint32_t d) { return NULL; } -static void* unpack_uint64(void* data, uint64_t d) { return NULL; } -static void* unpack_int8(void* data, int8_t d) { return NULL; } -static void* unpack_int16(void* data, int16_t d) { return NULL; } -static void* unpack_int32(void* data, int32_t d) { return NULL; } -static void* unpack_int64(void* data, int64_t d) { return NULL; } -static void* unpack_float(void* data, float d) { return NULL; } -static void* unpack_double(void* data, double d) { return NULL; } -static void* unpack_nil(void* data) { return NULL; } -static void* unpack_true(void* data) { return NULL; } -static void* unpack_false(void* data) { return NULL; } -static void* unpack_array(void* data, unsigned int n) { return NULL; } -static void unpack_array_item(void* data, void* c, void* o) { } -static void* unpack_map(void* data, unsigned int n) { return NULL; } -static void unpack_map_item(void* data, void* c, void* k, void* v) { } -static void* unpack_raw(void* data, const char* b, const char* p, unsigned int l) { /*printf("unpack raw %p %lu\n",p,l);*/ return NULL; } - - -typedef struct { - size_t allocated; - size_t length; - char* buffer; -} pack_buffer; - -static const size_t PACK_INITIAL_BUFFER_SIZE = 32*1024; - -static void pack_buffer_init(pack_buffer* data) -{ - data->buffer = malloc(PACK_INITIAL_BUFFER_SIZE); - data->length = 0; - data->allocated = PACK_INITIAL_BUFFER_SIZE; -} - -static void pack_buffer_reset(pack_buffer* data) -{ - data->buffer = realloc(data->buffer, PACK_INITIAL_BUFFER_SIZE); - data->allocated = PACK_INITIAL_BUFFER_SIZE; - data->length = 0; -} - -static void pack_buffer_free(pack_buffer* data) -{ - free(data->buffer); -} - -static void pack_append_buffer(void* user, const char* b, unsigned int l) -{ - pack_buffer* data = (pack_buffer*)user; - if(data->allocated - data->length < l) { - data->buffer = realloc(data->buffer, data->allocated*2); - data->allocated *= 2; - } - memcpy(data->buffer + data->length, b, l); - data->length += l; -} - - -static const unsigned int TASK_INT_NUM = 1<<24; -static const unsigned int TASK_STR_LEN = 1<<15; -//static const unsigned int TASK_INT_NUM = 1<<20; -//static const unsigned int TASK_STR_LEN = 1<<12; -static const char* TASK_STR_PTR; - - -void bench_json(void) -{ - puts("== JSON =="); - - - yajl_gen_config gcfg = {0, NULL}; - yajl_gen g = yajl_gen_alloc(&gcfg); - - yajl_parser_config hcfg = { 0, 0 }; - yajl_callbacks callbacks = { - reformat_null, - reformat_boolean, - NULL, - NULL, - reformat_number, - reformat_string, - reformat_start_map, - reformat_map_key, - reformat_end_map, - reformat_start_array, - reformat_end_array - }; - yajl_handle h = yajl_alloc(&callbacks, &hcfg, NULL); - - - const unsigned char * buf; - unsigned int len; - - - puts("generate integer"); - reset_timer(); - { - unsigned int i; - yajl_gen_array_open(g); - for(i=0; i < TASK_INT_NUM; ++i) { - yajl_gen_integer(g, i); - } - yajl_gen_array_close(g); - } - show_timer(len); - - yajl_gen_get_buf(g, &buf, &len); - - puts("----"); - puts("parse integer"); - reset_timer(); - { - yajl_status stat = yajl_parse(h, buf, len); - if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { - unsigned char * str = yajl_get_error(h, 1, buf, len); - fprintf(stderr, (const char *) str); - } - } - show_timer(len); - - - //yajl_gen_clear(g); - yajl_gen_free(g); - g = yajl_gen_alloc(&gcfg); - yajl_free(h); - h = yajl_alloc(&callbacks, &hcfg, NULL); - - - puts("----"); - puts("generate string"); - reset_timer(); - { - unsigned int i; - yajl_gen_array_open(g); - for(i=0; i < TASK_STR_LEN; ++i) { - yajl_gen_string(g, (const unsigned char*)TASK_STR_PTR, i); - } - yajl_gen_array_close(g); - } - show_timer(len); - - yajl_gen_get_buf(g, &buf, &len); - - puts("----"); - puts("parse string"); - reset_timer(); - { - yajl_status stat = yajl_parse(h, buf, len); - if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { - unsigned char * str = yajl_get_error(h, 1, buf, len); - fprintf(stderr, (const char *) str); - } - } - show_timer(len); - - - yajl_gen_free(g); - yajl_free(h); -} - - -void bench_msgpack(void) -{ - puts("== MessagePack =="); - - - pack_buffer mpkbuf; - pack_buffer_init(&mpkbuf); - - msgpack_pack_t* mpk = msgpack_pack_new( - &mpkbuf, pack_append_buffer); - - msgpack_unpack_callback cb = { - unpack_uint8, - unpack_uint16, - unpack_uint32, - unpack_uint64, - unpack_int8, - unpack_int16, - unpack_int32, - unpack_int64, - unpack_float, - unpack_double, - unpack_nil, - unpack_true, - unpack_false, - unpack_array, - unpack_array_item, - unpack_map, - unpack_map_item, - unpack_raw, - }; - msgpack_unpack_t* mupk = msgpack_unpack_new(NULL, &cb); - - - size_t len; - const char* buf; - - - puts("pack integer"); - reset_timer(); - { - unsigned int i; - msgpack_pack_array(mpk, TASK_INT_NUM); - for(i=0; i < TASK_INT_NUM; ++i) { - msgpack_pack_unsigned_int(mpk, i); - } - } - show_timer(mpkbuf.length); - - len = mpkbuf.length; - buf = mpkbuf.buffer; - - puts("----"); - puts("unpack integer"); - reset_timer(); - { - size_t off = 0; - int ret = msgpack_unpack_execute(mupk, buf, len, &off); - if(ret < 0) { - fprintf(stderr, "Parse error.\n"); - } else if(ret == 0) { - fprintf(stderr, "Not finished.\n"); - } - } - show_timer(mpkbuf.length); - - - pack_buffer_reset(&mpkbuf); - msgpack_unpack_reset(mupk); - - - puts("----"); - puts("pack string"); - reset_timer(); - { - unsigned int i; - msgpack_pack_array(mpk, TASK_STR_LEN); - for(i=0; i < TASK_STR_LEN; ++i) { - msgpack_pack_raw(mpk, i); - msgpack_pack_raw_body(mpk, TASK_STR_PTR, i); - } - } - show_timer(mpkbuf.length); - - len = mpkbuf.length; - buf = mpkbuf.buffer; - - puts("----"); - puts("unpack string"); - reset_timer(); - { - size_t off = 0; - int ret = msgpack_unpack_execute(mupk, buf, len, &off); - if(ret < 0) { - fprintf(stderr, "Parse error.\n"); - } else if(ret == 0) { - fprintf(stderr, "Not finished.\n"); - } - } - show_timer(mpkbuf.length); - - - msgpack_unpack_free(mupk); - msgpack_pack_free(mpk); - pack_buffer_free(&mpkbuf); -} - -int main(int argc, char* argv[]) -{ - char* str = malloc(TASK_STR_LEN); - memset(str, 'a', TASK_STR_LEN); - TASK_STR_PTR = str; - - bench_msgpack(); - bench_json(); - - return 0; -} - - diff --git a/c/bench.mk b/c/bench.mk deleted file mode 100644 index c765e31..0000000 --- a/c/bench.mk +++ /dev/null @@ -1,9 +0,0 @@ - -CFLAGS += -Wall -g -I. -I.. -O4 -LDFLAGS += -lyajl - -all: bench - -bench: bench.o pack.o unpack.o pack.h unpack.h - $(CC) bench.o pack.o unpack.o $(CFLAGS) $(LDFLAGS) -o $@ - diff --git a/AUTHORS b/cpp/AUTHORS similarity index 100% rename from AUTHORS rename to cpp/AUTHORS diff --git a/COPYING b/cpp/COPYING similarity index 100% rename from COPYING rename to cpp/COPYING diff --git a/ChangeLog b/cpp/ChangeLog similarity index 100% rename from ChangeLog rename to cpp/ChangeLog diff --git a/LICENSE b/cpp/LICENSE similarity index 100% rename from LICENSE rename to cpp/LICENSE diff --git a/cpp/Makefile.am b/cpp/Makefile.am index c1b4981..aba7e28 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -1,9 +1,38 @@ -lib_LTLIBRARIES = libmsgpack.la +lib_LTLIBRARIES = libmsgpackc.la libmsgpack.la + +libmsgpackc_la_SOURCES = \ + unpack.c \ + object.c \ + vrefbuffer.c \ + zone.c + +# -version-info CURRENT:REVISION:AGE +libmsgpackc_la_LDFLAGS = -version-info 2:0:0 + libmsgpack_la_SOURCES = \ object.cpp +libmsgpack_la_LIBADD = -lmsgpackc + +# -version-info CURRENT:REVISION:AGE +libmsgpack_la_LDFLAGS = -version-info 2:0:0 + + nobase_include_HEADERS = \ + msgpack/pack_define.h \ + msgpack/pack_template.h \ + msgpack/unpack_define.h \ + msgpack/unpack_template.h \ + msgpack/sysdep.h \ + msgpack.h \ + msgpack/sbuffer.h \ + msgpack/vrefbuffer.h \ + msgpack/zbuffer.h \ + msgpack/pack.h \ + msgpack/unpack.h \ + msgpack/object.h \ + msgpack/zone.h \ msgpack.hpp \ msgpack/sbuffer.hpp \ msgpack/vrefbuffer.hpp \ @@ -30,16 +59,31 @@ nobase_include_HEADERS = \ msgpack/type/tr1/unordered_map.hpp \ msgpack/type/tr1/unordered_set.hpp -libmsgpack_la_LIBADD = -L../c -lmsgpackc -# -version-info CURRENT:REVISION:AGE -libmsgpack_la_LDFLAGS = -version-info 2:0:0 +# work around for duplicated object file name +libmsgpackc_la_CFLAGS = $(AM_CFLAGS) +libmsgpackc_la_CXXFLAGS = $(AM_CXXFLAGS) +libmsgpack_la_CFLAGS = $(AM_CFLAGS) +libmsgpack_la_CXXFLAGS = $(AM_CXXFLAGS) + + +EXTRA_DIST = \ + msgpack_vc8.vcproj \ + msgpack_vc8.sln \ + msgpack_vc8.postbuild.bat + check_PROGRAMS = \ - msgpack_test + msgpackc_test \ + msgpack_test -msgpack_test_SOURCES = test.cpp +msgpackc_test_SOURCES = msgpackc_test.cpp +msgpackc_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c +msgpackc_test_LDADD = libmsgpackc.la -lgtest_main + +msgpack_test_SOURCES = msgpack_test.cpp msgpack_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -I$(top_srcdir)/cpp msgpack_test_LDADD = libmsgpack.la -lgtest_main TESTS = $(check_PROGRAMS) + diff --git a/NOTICE b/cpp/NOTICE similarity index 100% rename from NOTICE rename to cpp/NOTICE diff --git a/cpp/bench.cpp b/cpp/bench.cpp deleted file mode 100644 index aa303fa..0000000 --- a/cpp/bench.cpp +++ /dev/null @@ -1,188 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -static const unsigned int TASK_INT_NUM = 1<<24; -static const unsigned int TASK_STR_LEN = 1<<15; -//static const unsigned int TASK_INT_NUM = 1<<22; -//static const unsigned int TASK_STR_LEN = 1<<13; -static const char* TASK_STR_PTR; - - -class simple_timer { -public: - void reset() { gettimeofday(&m_timeval, NULL); } - void show_stat(size_t bufsz) - { - struct timeval endtime; - gettimeofday(&endtime, NULL); - double sec = (endtime.tv_sec - m_timeval.tv_sec) - + (double)(endtime.tv_usec - m_timeval.tv_usec) / 1000 / 1000; - std::cout << sec << " sec" << std::endl; - std::cout << (double(bufsz)/1024/1024) << " MB" << std::endl; - std::cout << (bufsz/sec/1000/1000*8) << " Mbps" << std::endl; - } -private: - timeval m_timeval; -}; - - -class simple_buffer { -public: - static const size_t DEFAULT_INITIAL_SIZE = 32*1024;//512*1024*1024*2; - - simple_buffer(size_t initial_size = DEFAULT_INITIAL_SIZE) : - m_storage((char*)malloc(initial_size)), - m_allocated(initial_size), - m_used(0) - { - if(!m_storage) { throw std::bad_alloc(); } - } - - ~simple_buffer() - { - free(m_storage); - } - -public: - inline void write(const char* buf, size_t len) - { - if(m_allocated - m_used < len) { - expand_buffer(len); - } - memcpy(m_storage + m_used, buf, len); - m_used += len; - } - - void clear() - { - m_used = 0; - } - -private: - void expand_buffer(size_t req) - { - size_t nsize = m_allocated * 2; - size_t at_least = m_used + req; - while(nsize < at_least) { nsize *= 2; } - char* tmp = (char*)realloc(m_storage, nsize); - if(!tmp) { throw std::bad_alloc(); } - m_storage = tmp; - m_allocated = nsize; - } - -public: - size_t size() const { return m_used; } - const char* data() const { return m_storage; } - -private: - char* m_storage; - size_t m_allocated; - size_t m_used; -}; - - -void bench_msgpack_int() -{ - simple_buffer buf; - simple_timer timer; - - std::cout << "----" << std::endl; - std::cout << "pack integer" << std::endl; - - timer.reset(); - { - msgpack::packer pk(buf); - pk.pack_array(TASK_INT_NUM); - for(unsigned int i=0; i < TASK_INT_NUM; ++i) { - pk.pack_unsigned_int(i); - } - } - timer.show_stat(buf.size()); - - - std::cout << "----" << std::endl; - std::cout << "unpack integer" << std::endl; - - msgpack::zone z; - msgpack::object obj; - - timer.reset(); - { - obj = msgpack::unpack(buf.data(), buf.size(), z); - } - timer.show_stat(buf.size()); - - /* - std::cout << "----" << std::endl; - std::cout << "dynamic pack integer" << std::endl; - - buf.clear(); - - timer.reset(); - msgpack::pack(buf, obj); - timer.show_stat(buf.size()); - */ -} - -void bench_msgpack_str() -{ - simple_buffer buf; - simple_timer timer; - - std::cout << "----" << std::endl; - std::cout << "pack string" << std::endl; - - timer.reset(); - { - msgpack::packer pk(buf); - pk.pack_array(TASK_STR_LEN); - for(unsigned int i=0; i < TASK_STR_LEN; ++i) { - pk.pack_raw(i); - pk.pack_raw_body(TASK_STR_PTR, i); - } - } - timer.show_stat(buf.size()); - - - std::cout << "----" << std::endl; - std::cout << "unpack string" << std::endl; - - msgpack::zone z; - msgpack::object obj; - - timer.reset(); - { - obj = msgpack::unpack(buf.data(), buf.size(), z); - } - timer.show_stat(buf.size()); - - - /* - std::cout << "----" << std::endl; - std::cout << "dynamic pack string" << std::endl; - - buf.clear(); - - timer.reset(); - msgpack::pack(buf, obj); - timer.show_stat(buf.size()); - */ -} - -int main(void) -{ - char* str = (char*)malloc(TASK_STR_LEN); - memset(str, 'a', TASK_STR_LEN); - TASK_STR_PTR = str; - - bench_msgpack_int(); - bench_msgpack_str(); - - return 0; -} - diff --git a/cpp/bench.mk b/cpp/bench.mk deleted file mode 100644 index 8da2b7f..0000000 --- a/cpp/bench.mk +++ /dev/null @@ -1,9 +0,0 @@ - -CXXFLAGS += -Wall -g -I. -I.. -O4 -LDFLAGS += - -all: bench - -bench: bench.o unpack.o zone.o object.o pack.hpp unpack.hpp zone.hpp object.hpp - $(CXX) bench.o unpack.o zone.o object.o $(CXXFLAGS) $(LDFLAGS) -o $@ - diff --git a/bootstrap b/cpp/bootstrap similarity index 93% rename from bootstrap rename to cpp/bootstrap index 8ac504b..4a04e0a 100755 --- a/bootstrap +++ b/cpp/bootstrap @@ -32,7 +32,16 @@ fi mkdir -p ac -(cd cpp && ./preprocess.sh $@; cd ..) +test -f AUTHORS || touch AUTHORS +test -f COPYING || touch COPYING +test -f ChangeLog || touch ChangeLog +test -f NEWS || touch NEWS +test -f README || touch README + +if ! ./preprocess; then + exit 1 +fi + ACLOCAL="aclocal" diff --git a/configure.in b/cpp/configure.in similarity index 61% rename from configure.in rename to cpp/configure.in index 0f2c5d0..a4cb4d8 100644 --- a/configure.in +++ b/cpp/configure.in @@ -1,35 +1,21 @@ -AC_INIT(msgpack/unpack_template.h) +AC_INIT(object.cpp) AC_CONFIG_AUX_DIR(ac) AM_INIT_AUTOMAKE(msgpack, 0.4.3) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) -if test "" = "$CFLAGS"; then - CFLAGS="-g -O4" -fi +CFLAGS="-O4 -Wall $CFLAGS" + +AC_SUBST(CXXFLAGS) +CXXFLAGS="-O4 -Wall $CXXFLAGS" AC_PROG_CC - -CFLAGS="-O4 -Wall $CFLAGS -I.." - -AC_MSG_CHECKING([if c++ api is enabled]) -AC_ARG_ENABLE(cxx, - AS_HELP_STRING([--disable-cxx], - [don't build c++ api.]) ) -AC_MSG_RESULT($enable_cxx) -if test "$enable_cxx" != "no"; then - AC_SUBST(CXXFLAGS) - if test "" = "$CXXFLAGS"; then - CXXFLAGS="-g -O4" - fi -fi - -# FIXME enable_cxx AC_PROG_CXX -CXXFLAGS="-O4 -Wall $CXXFLAGS -I.. -I../c" +AC_PROG_LIBTOOL +AM_PROG_AS +AM_PROG_CC_C_O -# FIXME enable_cxx AC_LANG_PUSH([C++]) AC_CHECK_HEADERS(tr1/unordered_map) AC_CHECK_HEADERS(tr1/unordered_set) @@ -53,9 +39,5 @@ add CFLAGS="--march=i686" and CXXFLAGS="-march=i668" options to ./configure as f ]) fi -AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no") - -AC_PROG_LIBTOOL - -AC_OUTPUT([Makefile c/Makefile cpp/Makefile]) +AC_OUTPUT([Makefile]) diff --git a/c/msgpack.h b/cpp/msgpack.h similarity index 100% rename from c/msgpack.h rename to cpp/msgpack.h diff --git a/c/msgpack/object.h b/cpp/msgpack/object.h similarity index 100% rename from c/msgpack/object.h rename to cpp/msgpack/object.h diff --git a/c/msgpack/pack.h b/cpp/msgpack/pack.h similarity index 100% rename from c/msgpack/pack.h rename to cpp/msgpack/pack.h diff --git a/c/msgpack/sbuffer.h b/cpp/msgpack/sbuffer.h similarity index 100% rename from c/msgpack/sbuffer.h rename to cpp/msgpack/sbuffer.h diff --git a/c/msgpack/unpack.h b/cpp/msgpack/unpack.h similarity index 100% rename from c/msgpack/unpack.h rename to cpp/msgpack/unpack.h diff --git a/c/msgpack/vrefbuffer.h b/cpp/msgpack/vrefbuffer.h similarity index 100% rename from c/msgpack/vrefbuffer.h rename to cpp/msgpack/vrefbuffer.h diff --git a/c/msgpack/zbuffer.h b/cpp/msgpack/zbuffer.h similarity index 100% rename from c/msgpack/zbuffer.h rename to cpp/msgpack/zbuffer.h diff --git a/c/msgpack/zone.h b/cpp/msgpack/zone.h similarity index 100% rename from c/msgpack/zone.h rename to cpp/msgpack/zone.h diff --git a/cpp/test.cpp b/cpp/msgpack_test.cpp similarity index 100% rename from cpp/test.cpp rename to cpp/msgpack_test.cpp diff --git a/cpp/msgpack_vc8.postbuild.bat b/cpp/msgpack_vc8.postbuild.bat new file mode 100644 index 0000000..1bdfabe --- /dev/null +++ b/cpp/msgpack_vc8.postbuild.bat @@ -0,0 +1,49 @@ +IF NOT EXIST include MKDIR include +IF NOT EXIST include\msgpack MKDIR include\msgpack +IF NOT EXIST include\msgpack\type MKDIR include\msgpack\type +IF NOT EXIST include\msgpack\type\tr1 MKDIR include\msgpack\type\tr1 +IF EXIST bootstrap ( + copy ..\msgpack\pack_define.h include\msgpack\ + copy ..\msgpack\pack_template.h include\msgpack\ + copy ..\msgpack\unpack_define.h include\msgpack\ + copy ..\msgpack\unpack_template.h include\msgpack\ + copy ..\msgpack\sysdep.h include\msgpack\ +) ELSE ( + copy msgpack\pack_define.h include\msgpack\ + copy msgpack\pack_template.h include\msgpack\ + copy msgpack\unpack_define.h include\msgpack\ + copy msgpack\unpack_template.h include\msgpack\ + copy msgpack\sysdep.h include\msgpack\ +) +copy msgpack.h include\ +copy msgpack\sbuffer.h include\msgpack\ +copy msgpack\vrefbuffer.h include\msgpack\ +copy msgpack\pack.h include\msgpack\ +copy msgpack\unpack.h include\msgpack\ +copy msgpack\object.h include\msgpack\ +copy msgpack\zone.h include\msgpack\ +copy msgpack.hpp include\ +copy msgpack\sbuffer.hpp include\msgpack\ +copy msgpack\vrefbuffer.hpp include\msgpack\ +copy msgpack\pack.hpp include\msgpack\ +copy msgpack\unpack.hpp include\msgpack\ +copy msgpack\object.hpp include\msgpack\ +copy msgpack\zone.hpp include\msgpack\ +copy msgpack\type.hpp include\msgpack\type\ +copy msgpack\type\bool.hpp include\msgpack\type\ +copy msgpack\type\float.hpp include\msgpack\type\ +copy msgpack\type\int.hpp include\msgpack\type\ +copy msgpack\type\list.hpp include\msgpack\type\ +copy msgpack\type\deque.hpp include\msgpack\type\ +copy msgpack\type\map.hpp include\msgpack\type\ +copy msgpack\type\nil.hpp include\msgpack\type\ +copy msgpack\type\pair.hpp include\msgpack\type\ +copy msgpack\type\raw.hpp include\msgpack\type\ +copy msgpack\type\set.hpp include\msgpack\type\ +copy msgpack\type\string.hpp include\msgpack\type\ +copy msgpack\type\vector.hpp include\msgpack\type\ +copy msgpack\type\tuple.hpp include\msgpack\type\ +copy msgpack\type\define.hpp include\msgpack\type\ +copy msgpack\type\tr1\unordered_map.hpp include\msgpack\type\ +copy msgpack\type\tr1\unordered_set.hpp include\msgpack\type\ + diff --git a/msgpack_vc8.sln b/cpp/msgpack_vc8.sln similarity index 100% rename from msgpack_vc8.sln rename to cpp/msgpack_vc8.sln diff --git a/msgpack_vc8.vcproj b/cpp/msgpack_vc8.vcproj old mode 100755 new mode 100644 similarity index 100% rename from msgpack_vc8.vcproj rename to cpp/msgpack_vc8.vcproj diff --git a/c/test.cpp b/cpp/msgpackc_test.cpp similarity index 100% rename from c/test.cpp rename to cpp/msgpackc_test.cpp diff --git a/c/object.c b/cpp/object.c similarity index 100% rename from c/object.c rename to cpp/object.c diff --git a/cpp/preprocess.sh b/cpp/preprocess similarity index 62% rename from cpp/preprocess.sh rename to cpp/preprocess index 2e06c10..63af4c6 100755 --- a/cpp/preprocess.sh +++ b/cpp/preprocess @@ -14,4 +14,8 @@ preprocess() { preprocess msgpack/type/tuple.hpp preprocess msgpack/type/define.hpp preprocess msgpack/zone.hpp +cp -f ../msgpack/pack_define.h msgpack/ +cp -f ../msgpack/pack_template.h msgpack/ +cp -f ../msgpack/unpack_define.h msgpack/ +cp -f ../msgpack/unpack_template.h msgpack/ diff --git a/c/unpack.c b/cpp/unpack.c similarity index 100% rename from c/unpack.c rename to cpp/unpack.c diff --git a/c/vrefbuffer.c b/cpp/vrefbuffer.c similarity index 100% rename from c/vrefbuffer.c rename to cpp/vrefbuffer.c diff --git a/c/zone.c b/cpp/zone.c similarity index 100% rename from c/zone.c rename to cpp/zone.c diff --git a/msgpack_vc8.postbuild.bat b/msgpack_vc8.postbuild.bat deleted file mode 100644 index 7ee2586..0000000 --- a/msgpack_vc8.postbuild.bat +++ /dev/null @@ -1,41 +0,0 @@ -IF NOT EXIST include MKDIR include -IF NOT EXIST include\msgpack MKDIR include\msgpack -IF NOT EXIST include\msgpack\type MKDIR include\msgpack\type -IF NOT EXIST include\msgpack\type\tr1 MKDIR include\msgpack\type\tr1 -copy msgpack\pack_define.h include\msgpack\ -copy msgpack\pack_template.h include\msgpack\ -copy msgpack\unpack_define.h include\msgpack\ -copy msgpack\unpack_template.h include\msgpack\ -copy msgpack\sysdep.h include\msgpack\ -copy c\msgpack.h include\ -copy c\msgpack\sbuffer.h include\msgpack\ -copy c\msgpack\vrefbuffer.h include\msgpack\ -copy c\msgpack\pack.h include\msgpack\ -copy c\msgpack\unpack.h include\msgpack\ -copy c\msgpack\object.h include\msgpack\ -copy c\msgpack\zone.h include\msgpack\ -copy cpp\msgpack.hpp include\ -copy cpp\msgpack\sbuffer.hpp include\msgpack\ -copy cpp\msgpack\vrefbuffer.hpp include\msgpack\ -copy cpp\msgpack\pack.hpp include\msgpack\ -copy cpp\msgpack\unpack.hpp include\msgpack\ -copy cpp\msgpack\object.hpp include\msgpack\ -copy cpp\msgpack\zone.hpp include\msgpack\ -copy cpp\msgpack\type.hpp include\msgpack\type\ -copy cpp\msgpack\type\bool.hpp include\msgpack\type\ -copy cpp\msgpack\type\float.hpp include\msgpack\type\ -copy cpp\msgpack\type\int.hpp include\msgpack\type\ -copy cpp\msgpack\type\list.hpp include\msgpack\type\ -copy cpp\msgpack\type\deque.hpp include\msgpack\type\ -copy cpp\msgpack\type\map.hpp include\msgpack\type\ -copy cpp\msgpack\type\nil.hpp include\msgpack\type\ -copy cpp\msgpack\type\pair.hpp include\msgpack\type\ -copy cpp\msgpack\type\raw.hpp include\msgpack\type\ -copy cpp\msgpack\type\set.hpp include\msgpack\type\ -copy cpp\msgpack\type\string.hpp include\msgpack\type\ -copy cpp\msgpack\type\vector.hpp include\msgpack\type\ -copy cpp\msgpack\type\tuple.hpp include\msgpack\type\ -copy cpp\msgpack\type\define.hpp include\msgpack\type\ -copy cpp\msgpack\type\tr1\unordered_map.hpp include\msgpack\type\ -copy cpp\msgpack\type\tr1\unordered_set.hpp include\msgpack\type\ - From 58854fdae90e428d9ab18bc3f457f1eb0b447446 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 18 Apr 2010 00:39:23 +0900 Subject: [PATCH 0409/1172] cpp: add cpp/README.md --- cpp/README.md | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 cpp/README.md diff --git a/cpp/README.md b/cpp/README.md new file mode 100644 index 0000000..cc1ba38 --- /dev/null +++ b/cpp/README.md @@ -0,0 +1,78 @@ +MessagePack for C/C++ +===================== +Binary-based efficient object serialization library. + + +## Installation + +Download latest package from [releases of MessagePack](http://sourceforge.net/projects/msgpack/files/) and extract it. + +On UNIX-like platform, run ./configure && make && sudo make install: + + $ ./configure + $ make + $ sudo make install + +On Windows, open msgpack_vc8.vcproj file and build it using batch build. DLLs are built on lib folder, and the headers are built on include folder. + +To use the library in your program, include msgpack.hpp header and link msgpack and msgpackc library. + + +## Example + + #include + #include + + int main(void) { + // This is target object. + std::vector target; + target.push_back("Hello,"); + target.push_back("World!"); + + // Serialize it. + msgpack::sbuffer buffer; // simple buffer + msgpack::pack(&buffer, target); + + // Deserialize the serialized data. + msgpack::zone mempool; // this manages the life of deserialized object + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(buffer.data, buffer.size, NULL, &mempool, &obj); + + if(ret != msgapck::UNPACK_SUCCESS) { + // error check + exit(1); + } + + // Print the deserialized object to stdout. + std::cout << obj << std::endl; // ["Hello," "World!"] + + // Convert the deserialized object to staticaly typed object. + std::vector result; + obj.convert(&result); + + // If the type is mismatched, it throws msgpack::type_error. + obj.as(); // type is mismatched, msgpack::type_error is thrown + } + +API document and other example codes are available at the [wiki.](http://msgpack.sourceforge.net/start) + + +## License + +Copyright (C) 2008-2010 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +See also NOTICE file. + From 05b8c00ee78b5806814f133f6ccd33148d1e13a0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 18 Apr 2010 00:39:45 +0900 Subject: [PATCH 0410/1172] cpp: fixes windows compatibility --- cpp/Makefile.am | 6 +++--- cpp/bootstrap | 2 +- cpp/msgpack_test.cpp | 4 ++-- cpp/msgpack_vc8.postbuild.bat | 20 +++++++------------- cpp/msgpack_vc8.vcproj | 18 +++++++++--------- cpp/preprocess | 1 + cpp/unpack.c | 2 +- 7 files changed, 24 insertions(+), 29 deletions(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index aba7e28..a297ba7 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -68,21 +68,21 @@ libmsgpack_la_CXXFLAGS = $(AM_CXXFLAGS) EXTRA_DIST = \ + README.md \ + LICENSE \ + NOTICE \ msgpack_vc8.vcproj \ msgpack_vc8.sln \ msgpack_vc8.postbuild.bat - check_PROGRAMS = \ msgpackc_test \ msgpack_test msgpackc_test_SOURCES = msgpackc_test.cpp -msgpackc_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c msgpackc_test_LDADD = libmsgpackc.la -lgtest_main msgpack_test_SOURCES = msgpack_test.cpp -msgpack_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -I$(top_srcdir)/cpp msgpack_test_LDADD = libmsgpack.la -lgtest_main TESTS = $(check_PROGRAMS) diff --git a/cpp/bootstrap b/cpp/bootstrap index 4a04e0a..a95c304 100755 --- a/cpp/bootstrap +++ b/cpp/bootstrap @@ -36,7 +36,7 @@ test -f AUTHORS || touch AUTHORS test -f COPYING || touch COPYING test -f ChangeLog || touch ChangeLog test -f NEWS || touch NEWS -test -f README || touch README +test -f README || cp -f README.md README if ! ./preprocess; then exit 1 diff --git a/cpp/msgpack_test.cpp b/cpp/msgpack_test.cpp index 113914a..0dd0ffc 100644 --- a/cpp/msgpack_test.cpp +++ b/cpp/msgpack_test.cpp @@ -438,7 +438,7 @@ TEST(MSGPACK_STL, simple_buffer_multiset) #ifdef HAVE_TR1_UNORDERED_MAP #include -#include "cpp/msgpack/type/tr1/unordered_map.hpp" +#include "msgpack/type/tr1/unordered_map.hpp" TEST(MSGPACK_TR1, simple_buffer_unordered_map) { for (unsigned int k = 0; k < kLoop; k++) { @@ -499,7 +499,7 @@ TEST(MSGPACK_TR1, simple_buffer_unordered_multimap) #ifdef HAVE_TR1_UNORDERED_SET #include -#include "cpp/msgpack/type/tr1/unordered_set.hpp" +#include "msgpack/type/tr1/unordered_set.hpp" TEST(MSGPACK_TR1, simple_buffer_unordered_set) { for (unsigned int k = 0; k < kLoop; k++) { diff --git a/cpp/msgpack_vc8.postbuild.bat b/cpp/msgpack_vc8.postbuild.bat index 1bdfabe..bae13f3 100644 --- a/cpp/msgpack_vc8.postbuild.bat +++ b/cpp/msgpack_vc8.postbuild.bat @@ -2,22 +2,15 @@ IF NOT EXIST include MKDIR include IF NOT EXIST include\msgpack MKDIR include\msgpack IF NOT EXIST include\msgpack\type MKDIR include\msgpack\type IF NOT EXIST include\msgpack\type\tr1 MKDIR include\msgpack\type\tr1 -IF EXIST bootstrap ( - copy ..\msgpack\pack_define.h include\msgpack\ - copy ..\msgpack\pack_template.h include\msgpack\ - copy ..\msgpack\unpack_define.h include\msgpack\ - copy ..\msgpack\unpack_template.h include\msgpack\ - copy ..\msgpack\sysdep.h include\msgpack\ -) ELSE ( - copy msgpack\pack_define.h include\msgpack\ - copy msgpack\pack_template.h include\msgpack\ - copy msgpack\unpack_define.h include\msgpack\ - copy msgpack\unpack_template.h include\msgpack\ - copy msgpack\sysdep.h include\msgpack\ -) +copy msgpack\pack_define.h include\msgpack\ +copy msgpack\pack_template.h include\msgpack\ +copy msgpack\unpack_define.h include\msgpack\ +copy msgpack\unpack_template.h include\msgpack\ +copy msgpack\sysdep.h include\msgpack\ copy msgpack.h include\ copy msgpack\sbuffer.h include\msgpack\ copy msgpack\vrefbuffer.h include\msgpack\ +copy msgpack\zbuffer.h include\msgpack\ copy msgpack\pack.h include\msgpack\ copy msgpack\unpack.h include\msgpack\ copy msgpack\object.h include\msgpack\ @@ -25,6 +18,7 @@ copy msgpack\zone.h include\msgpack\ copy msgpack.hpp include\ copy msgpack\sbuffer.hpp include\msgpack\ copy msgpack\vrefbuffer.hpp include\msgpack\ +copy msgpack\zbuffer.hpp include\msgpack\ copy msgpack\pack.hpp include\msgpack\ copy msgpack\unpack.hpp include\msgpack\ copy msgpack\object.hpp include\msgpack\ diff --git a/cpp/msgpack_vc8.vcproj b/cpp/msgpack_vc8.vcproj index a3fa28f..2a0eebe 100644 --- a/cpp/msgpack_vc8.vcproj +++ b/cpp/msgpack_vc8.vcproj @@ -43,7 +43,7 @@ z); - template_destroy(mpac->ctx); + template_destroy(CTX_CAST(mpac->ctx)); free(mpac->ctx); decl_count(mpac->buffer); } From fb96617377ed7330edcf239d807d2ae378e336e9 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sun, 18 Apr 2010 01:18:40 +0900 Subject: [PATCH 0411/1172] java: add javadoc,javadoc-jar,pom,dist,mvn-install,mvn-deploy ANT tasks. Now maven2 repository for msgpack is created at the following URL. - http://msgpack.sourceforge.net/maven2/ --- java/build.xml | 127 +++++++++++++++++++++++++++++++++++++++++++++++-- java/ivy.xml | 2 +- 2 files changed, 125 insertions(+), 4 deletions(-) diff --git a/java/build.xml b/java/build.xml index 52c5218..f6e93ae 100644 --- a/java/build.xml +++ b/java/build.xml @@ -2,6 +2,12 @@ xmlns:ivy="antlib:org.apache.ivy.ant" xmlns:mvn="urn:maven-artifact-ant"> + + + + + + @@ -10,7 +16,11 @@ + + + + @@ -27,6 +37,10 @@ + + + - - - @@ -179,4 +190,114 @@ + + + + + +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + Javadoc warnings! + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/java/ivy.xml b/java/ivy.xml index a694214..0e07cca 100644 --- a/java/ivy.xml +++ b/java/ivy.xml @@ -1,7 +1,7 @@ - MessagePack From f53c351fd28b3bea6a03416a54aff631499af65a Mon Sep 17 00:00:00 2001 From: Hideyuki Tanaka Date: Sun, 18 Apr 2010 02:17:49 +0900 Subject: [PATCH 0412/1172] haskell binding --- haskell/LICENSE | 24 + haskell/README | 0 haskell/Setup.lhs | 3 + haskell/cbits/msgpack.c | 137 ++++++ haskell/msgpack.cabal | 32 ++ haskell/src/Data/MessagePack.hs | 63 +++ haskell/src/Data/MessagePack/Base.hsc | 581 +++++++++++++++++++++++++ haskell/src/Data/MessagePack/Class.hs | 97 +++++ haskell/src/Data/MessagePack/Feed.hs | 59 +++ haskell/src/Data/MessagePack/Monad.hs | 153 +++++++ haskell/src/Data/MessagePack/Stream.hs | 84 ++++ haskell/test/Monad.hs | 16 + haskell/test/Stream.hs | 14 + haskell/test/Test.hs | 36 ++ 14 files changed, 1299 insertions(+) create mode 100644 haskell/LICENSE create mode 100644 haskell/README create mode 100644 haskell/Setup.lhs create mode 100644 haskell/cbits/msgpack.c create mode 100644 haskell/msgpack.cabal create mode 100644 haskell/src/Data/MessagePack.hs create mode 100644 haskell/src/Data/MessagePack/Base.hsc create mode 100644 haskell/src/Data/MessagePack/Class.hs create mode 100644 haskell/src/Data/MessagePack/Feed.hs create mode 100644 haskell/src/Data/MessagePack/Monad.hs create mode 100644 haskell/src/Data/MessagePack/Stream.hs create mode 100644 haskell/test/Monad.hs create mode 100644 haskell/test/Stream.hs create mode 100644 haskell/test/Test.hs diff --git a/haskell/LICENSE b/haskell/LICENSE new file mode 100644 index 0000000..2de30f6 --- /dev/null +++ b/haskell/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2009, Hideyuki Tanaka +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the Hideyuki Tanaka nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY Hideyuki Tanaka ''AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/haskell/README b/haskell/README new file mode 100644 index 0000000..e69de29 diff --git a/haskell/Setup.lhs b/haskell/Setup.lhs new file mode 100644 index 0000000..5bde0de --- /dev/null +++ b/haskell/Setup.lhs @@ -0,0 +1,3 @@ +#!/usr/bin/env runhaskell +> import Distribution.Simple +> main = defaultMain diff --git a/haskell/cbits/msgpack.c b/haskell/cbits/msgpack.c new file mode 100644 index 0000000..be44592 --- /dev/null +++ b/haskell/cbits/msgpack.c @@ -0,0 +1,137 @@ +#include + +void msgpack_sbuffer_init_wrap(msgpack_sbuffer* sbuf) +{ + msgpack_sbuffer_init(sbuf); +} + +void msgpack_sbuffer_destroy_wrap(msgpack_sbuffer* sbuf) +{ + msgpack_sbuffer_destroy(sbuf); +} + +int msgpack_sbuffer_write_wrap(void* data, const char* buf, unsigned int len) +{ + return msgpack_sbuffer_write(data, buf, len); +} + +msgpack_packer* msgpack_packer_new_wrap(void *data, msgpack_packer_write callback) +{ + return msgpack_packer_new(data, callback); +} + +void msgpack_packer_free_wrap(msgpack_packer* pk) +{ + msgpack_packer_free(pk); +} + +int msgpack_pack_uint8_wrap(msgpack_packer* pk, uint8_t d) +{ + return msgpack_pack_uint8(pk, d); +} + +int msgpack_pack_uint16_wrap(msgpack_packer* pk, uint16_t d) +{ + return msgpack_pack_uint16(pk, d); +} + +int msgpack_pack_uint32_wrap(msgpack_packer* pk, uint32_t d) +{ + return msgpack_pack_uint32(pk, d); +} + +int msgpack_pack_uint64_wrap(msgpack_packer* pk, uint64_t d) +{ + return msgpack_pack_uint64(pk, d); +} + +int msgpack_pack_int8_wrap(msgpack_packer* pk, int8_t d) +{ + return msgpack_pack_int8(pk, d); +} + +int msgpack_pack_int16_wrap(msgpack_packer* pk, int16_t d) +{ + return msgpack_pack_int16(pk, d); +} + +int msgpack_pack_int32_wrap(msgpack_packer* pk, int32_t d) +{ + return msgpack_pack_int32(pk, d); +} + +int msgpack_pack_int64_wrap(msgpack_packer* pk, int64_t d) +{ + return msgpack_pack_int64(pk, d); +} + +int msgpack_pack_double_wrap(msgpack_packer* pk, double d) +{ + return msgpack_pack_double(pk, d); +} + +int msgpack_pack_nil_wrap(msgpack_packer* pk) +{ + return msgpack_pack_nil(pk); +} + +int msgpack_pack_true_wrap(msgpack_packer* pk) +{ + return msgpack_pack_true(pk); +} + +int msgpack_pack_false_wrap(msgpack_packer* pk) +{ + return msgpack_pack_false(pk); +} + +int msgpack_pack_array_wrap(msgpack_packer* pk, unsigned int n) +{ + return msgpack_pack_array(pk, n); +} + +int msgpack_pack_map_wrap(msgpack_packer* pk, unsigned int n) +{ + return msgpack_pack_map(pk, n); +} + +int msgpack_pack_raw_wrap(msgpack_packer* pk, size_t l) +{ + return msgpack_pack_raw(pk, l); +} + +int msgpack_pack_raw_body_wrap(msgpack_packer* pk, const void *b, size_t l) +{ + return msgpack_pack_raw_body(pk, b, l); +} + +bool msgpack_unpacker_reserve_buffer_wrap(msgpack_unpacker *mpac, size_t size) +{ + return msgpack_unpacker_reserve_buffer(mpac, size); +} + +char *msgpack_unpacker_buffer_wrap(msgpack_unpacker *mpac) +{ + return msgpack_unpacker_buffer(mpac); +} + +size_t msgpack_unpacker_buffer_capacity_wrap(const msgpack_unpacker *mpac) +{ + return msgpack_unpacker_buffer_capacity(mpac); +} + +void msgpack_unpacker_buffer_consumed_wrap(msgpack_unpacker *mpac, size_t size) +{ + msgpack_unpacker_buffer_consumed(mpac, size); +} + +void msgpack_unpacker_data_wrap(msgpack_unpacker *mpac, msgpack_object *obj) +{ + *obj=msgpack_unpacker_data(mpac); +} + +size_t msgpack_unpacker_message_size_wrap(const msgpack_unpacker *mpac) +{ + return msgpack_unpacker_message_size(mpac); +} + diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal new file mode 100644 index 0000000..505a2b9 --- /dev/null +++ b/haskell/msgpack.cabal @@ -0,0 +1,32 @@ +Name: msgpack +Version: 0.2.0 +License: BSD3 +License-File: LICENSE +Author: Hideyuki Tanaka +Maintainer: Hideyuki Tanaka +Category: Data +Synopsis: A Haskell binding to MessagePack +Description: + A Haskell binding to MessagePack +Homepage: http://github.com/tanakh/hsmsgpack +Stability: Experimental +Tested-with: GHC==6.10.4 +Cabal-Version: >=1.2 +Build-Type: Simple + +library + build-depends: base>=4 && <5, mtl, bytestring + ghc-options: -O2 -Wall + hs-source-dirs: src + extra-libraries: msgpackc + + Exposed-modules: + Data.MessagePack + Data.MessagePack.Base + Data.MessagePack.Class + Data.MessagePack.Feed + Data.MessagePack.Monad + Data.MessagePack.Stream + + C-Sources: + cbits/msgpack.c diff --git a/haskell/src/Data/MessagePack.hs b/haskell/src/Data/MessagePack.hs new file mode 100644 index 0000000..2949e60 --- /dev/null +++ b/haskell/src/Data/MessagePack.hs @@ -0,0 +1,63 @@ +-------------------------------------------------------------------- +-- | +-- Module : Data.MessagePack +-- Copyright : (c) Hideyuki Tanaka, 2009 +-- License : BSD3 +-- +-- Maintainer: tanaka.hideyuki@gmail.com +-- Stability : experimental +-- Portability: portable +-- +-- Simple interface to pack and unpack MessagePack data. +-- +-------------------------------------------------------------------- + +module Data.MessagePack( + module Data.MessagePack.Base, + module Data.MessagePack.Class, + module Data.MessagePack.Feed, + module Data.MessagePack.Monad, + module Data.MessagePack.Stream, + + -- * Pack and Unpack + packb, + unpackb, + + -- * Pure version of Pack and Unpack + packb', + unpackb', + ) where + +import Data.ByteString (ByteString) +import System.IO.Unsafe + +import Data.MessagePack.Base +import Data.MessagePack.Class +import Data.MessagePack.Feed +import Data.MessagePack.Monad +import Data.MessagePack.Stream + +-- | Pack Haskell data to MessagePack string. +packb :: OBJECT a => a -> IO ByteString +packb dat = do + sb <- newSimpleBuffer + pc <- newPacker sb + pack pc dat + simpleBufferData sb + +-- | Unpack MessagePack string to Haskell data. +unpackb :: OBJECT a => ByteString -> IO (Result a) +unpackb bs = do + withZone $ \z -> do + r <- unpackObject z bs + return $ case r of + Left err -> Left (show err) + Right (_, dat) -> fromObject dat + +-- | Pure version of 'packb'. +packb' :: OBJECT a => a -> ByteString +packb' dat = unsafePerformIO $ packb dat + +-- | Pure version of 'unpackb'. +unpackb' :: OBJECT a => ByteString -> Result a +unpackb' bs = unsafePerformIO $ unpackb bs diff --git a/haskell/src/Data/MessagePack/Base.hsc b/haskell/src/Data/MessagePack/Base.hsc new file mode 100644 index 0000000..ad71712 --- /dev/null +++ b/haskell/src/Data/MessagePack/Base.hsc @@ -0,0 +1,581 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE ForeignFunctionInterface #-} + +-------------------------------------------------------------------- +-- | +-- Module : Data.MessagePack.Base +-- Copyright : (c) Hideyuki Tanaka, 2009 +-- License : BSD3 +-- +-- Maintainer: tanaka.hideyuki@gmail.com +-- Stability : experimental +-- Portability: portable +-- +-- Low Level Interface to MessagePack C API +-- +-------------------------------------------------------------------- + +module Data.MessagePack.Base( + -- * Simple Buffer + SimpleBuffer, + newSimpleBuffer, + simpleBufferData, + + -- * Serializer + Packer, + newPacker, + + packU8, + packU16, + packU32, + packU64, + packS8, + packS16, + packS32, + packS64, + + packTrue, + packFalse, + + packInt, + packDouble, + packNil, + packBool, + + packArray, + packMap, + packRAW, + packRAWBody, + packRAW', + + -- * Stream Deserializer + Unpacker, + defaultInitialBufferSize, + newUnpacker, + unpackerReserveBuffer, + unpackerBuffer, + unpackerBufferCapacity, + unpackerBufferConsumed, + unpackerFeed, + unpackerExecute, + unpackerData, + unpackerReleaseZone, + unpackerResetZone, + unpackerReset, + unpackerMessageSize, + + -- * MessagePack Object + Object(..), + packObject, + + UnpackReturn(..), + unpackObject, + + -- * Memory Zone + Zone, + newZone, + freeZone, + withZone, + ) where + +import Control.Exception +import Control.Monad +import Data.ByteString (ByteString) +import qualified Data.ByteString as BS hiding (pack, unpack) +import Data.Int +import Data.Word +import Foreign.C +import Foreign.Concurrent +import Foreign.ForeignPtr hiding (newForeignPtr) +import Foreign.Marshal.Alloc +import Foreign.Marshal.Array +import Foreign.Ptr +import Foreign.Storable + +#include + +type SimpleBuffer = ForeignPtr () + +type WriteCallback = Ptr () -> CString -> CUInt -> IO CInt + +-- | Create a new Simple Buffer. It will be deleted automatically. +newSimpleBuffer :: IO SimpleBuffer +newSimpleBuffer = do + ptr <- mallocBytes (#size msgpack_sbuffer) + fptr <- newForeignPtr ptr $ do + msgpack_sbuffer_destroy ptr + free ptr + withForeignPtr fptr $ \p -> + msgpack_sbuffer_init p + return fptr + +-- | Get data of Simple Buffer. +simpleBufferData :: SimpleBuffer -> IO ByteString +simpleBufferData sb = + withForeignPtr sb $ \ptr -> do + size <- (#peek msgpack_sbuffer, size) ptr + dat <- (#peek msgpack_sbuffer, data) ptr + BS.packCStringLen (dat, fromIntegral (size :: CSize)) + +foreign import ccall "msgpack_sbuffer_init_wrap" msgpack_sbuffer_init :: + Ptr () -> IO () + +foreign import ccall "msgpack_sbuffer_destroy_wrap" msgpack_sbuffer_destroy :: + Ptr () -> IO () + +foreign import ccall "msgpack_sbuffer_write_wrap" msgpack_sbuffer_write :: + WriteCallback + +type Packer = ForeignPtr () + +-- | Create new Packer. It will be deleted automatically. +newPacker :: SimpleBuffer -> IO Packer +newPacker sbuf = do + cb <- wrap_callback msgpack_sbuffer_write + ptr <- withForeignPtr sbuf $ \ptr -> + msgpack_packer_new ptr cb + fptr <- newForeignPtr ptr $ do + msgpack_packer_free ptr + return fptr + +foreign import ccall "msgpack_packer_new_wrap" msgpack_packer_new :: + Ptr () -> FunPtr WriteCallback -> IO (Ptr ()) + +foreign import ccall "msgpack_packer_free_wrap" msgpack_packer_free :: + Ptr () -> IO () + +foreign import ccall "wrapper" wrap_callback :: + WriteCallback -> IO (FunPtr WriteCallback) + +packU8 :: Packer -> Word8 -> IO Int +packU8 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_uint8 ptr n + +foreign import ccall "msgpack_pack_uint8_wrap" msgpack_pack_uint8 :: + Ptr () -> Word8 -> IO CInt + +packU16 :: Packer -> Word16 -> IO Int +packU16 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_uint16 ptr n + +foreign import ccall "msgpack_pack_uint16_wrap" msgpack_pack_uint16 :: + Ptr () -> Word16 -> IO CInt + +packU32 :: Packer -> Word32 -> IO Int +packU32 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_uint32 ptr n + +foreign import ccall "msgpack_pack_uint32_wrap" msgpack_pack_uint32 :: + Ptr () -> Word32 -> IO CInt + +packU64 :: Packer -> Word64 -> IO Int +packU64 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_uint64 ptr n + +foreign import ccall "msgpack_pack_uint64_wrap" msgpack_pack_uint64 :: + Ptr () -> Word64 -> IO CInt + +packS8 :: Packer -> Int8 -> IO Int +packS8 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_int8 ptr n + +foreign import ccall "msgpack_pack_int8_wrap" msgpack_pack_int8 :: + Ptr () -> Int8 -> IO CInt + +packS16 :: Packer -> Int16 -> IO Int +packS16 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_int16 ptr n + +foreign import ccall "msgpack_pack_int16_wrap" msgpack_pack_int16 :: + Ptr () -> Int16 -> IO CInt + +packS32 :: Packer -> Int32 -> IO Int +packS32 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_int32 ptr n + +foreign import ccall "msgpack_pack_int32_wrap" msgpack_pack_int32 :: + Ptr () -> Int32 -> IO CInt + +packS64 :: Packer -> Int64 -> IO Int +packS64 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_int64 ptr n + +foreign import ccall "msgpack_pack_int64_wrap" msgpack_pack_int64 :: + Ptr () -> Int64 -> IO CInt + +-- | Pack an integral data. +packInt :: Integral a => Packer -> a -> IO Int +packInt pc n = packS64 pc $ fromIntegral n + +-- | Pack a double data. +packDouble :: Packer -> Double -> IO Int +packDouble pc d = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_double ptr (realToFrac d) + +foreign import ccall "msgpack_pack_double_wrap" msgpack_pack_double :: + Ptr () -> CDouble -> IO CInt + +-- | Pack a nil. +packNil :: Packer -> IO Int +packNil pc = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_nil ptr + +foreign import ccall "msgpack_pack_nil_wrap" msgpack_pack_nil :: + Ptr () -> IO CInt + +packTrue :: Packer -> IO Int +packTrue pc = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_true ptr + +foreign import ccall "msgpack_pack_true_wrap" msgpack_pack_true :: + Ptr () -> IO CInt + +packFalse :: Packer -> IO Int +packFalse pc = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_false ptr + +foreign import ccall "msgpack_pack_false_wrap" msgpack_pack_false :: + Ptr () -> IO CInt + +-- | Pack a bool data. +packBool :: Packer -> Bool -> IO Int +packBool pc True = packTrue pc +packBool pc False = packFalse pc + +-- | 'packArray' @p n@ starts packing an array. +-- Next @n@ data will consist this array. +packArray :: Packer -> Int -> IO Int +packArray pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_array ptr (fromIntegral n) + +foreign import ccall "msgpack_pack_array_wrap" msgpack_pack_array :: + Ptr () -> CUInt -> IO CInt + +-- | 'packMap' @p n@ starts packing a map. +-- Next @n@ pairs of data (2*n data) will consist this map. +packMap :: Packer -> Int -> IO Int +packMap pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_map ptr (fromIntegral n) + +foreign import ccall "msgpack_pack_map_wrap" msgpack_pack_map :: + Ptr () -> CUInt -> IO CInt + +-- | 'packRAW' @p n@ starts packing a byte sequence. +-- Next total @n@ bytes of 'packRAWBody' call will consist this sequence. +packRAW :: Packer -> Int -> IO Int +packRAW pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_raw ptr (fromIntegral n) + +foreign import ccall "msgpack_pack_raw_wrap" msgpack_pack_raw :: + Ptr () -> CSize -> IO CInt + +-- | Pack a byte sequence. +packRAWBody :: Packer -> ByteString -> IO Int +packRAWBody pc bs = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + BS.useAsCStringLen bs $ \(str, len) -> + msgpack_pack_raw_body ptr (castPtr str) (fromIntegral len) + +foreign import ccall "msgpack_pack_raw_body_wrap" msgpack_pack_raw_body :: + Ptr () -> Ptr () -> CSize -> IO CInt + +-- | Pack a single byte stream. It calls 'packRAW' and 'packRAWBody'. +packRAW' :: Packer -> ByteString -> IO Int +packRAW' pc bs = do + packRAW pc (BS.length bs) + packRAWBody pc bs + +type Unpacker = ForeignPtr () + +defaultInitialBufferSize :: Int +defaultInitialBufferSize = 32 * 1024 -- #const MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE + +-- | 'newUnpacker' @initialBufferSize@ creates a new Unpacker. It will be deleted automatically. +newUnpacker :: Int -> IO Unpacker +newUnpacker initialBufferSize = do + ptr <- msgpack_unpacker_new (fromIntegral initialBufferSize) + fptr <- newForeignPtr ptr $ do + msgpack_unpacker_free ptr + return fptr + +foreign import ccall "msgpack_unpacker_new" msgpack_unpacker_new :: + CSize -> IO (Ptr ()) + +foreign import ccall "msgpack_unpacker_free" msgpack_unpacker_free :: + Ptr() -> IO () + +-- | 'unpackerReserveBuffer' @up size@ reserves at least @size@ bytes of buffer. +unpackerReserveBuffer :: Unpacker -> Int -> IO Bool +unpackerReserveBuffer up size = + withForeignPtr up $ \ptr -> + liftM (/=0) $ msgpack_unpacker_reserve_buffer ptr (fromIntegral size) + +foreign import ccall "msgpack_unpacker_reserve_buffer_wrap" msgpack_unpacker_reserve_buffer :: + Ptr () -> CSize -> IO CChar + +-- | Get a pointer of unpacker buffer. +unpackerBuffer :: Unpacker -> IO (Ptr CChar) +unpackerBuffer up = + withForeignPtr up $ \ptr -> + msgpack_unpacker_buffer ptr + +foreign import ccall "msgpack_unpacker_buffer_wrap" msgpack_unpacker_buffer :: + Ptr () -> IO (Ptr CChar) + +-- | Get size of allocated buffer. +unpackerBufferCapacity :: Unpacker -> IO Int +unpackerBufferCapacity up = + withForeignPtr up $ \ptr -> + liftM fromIntegral $ msgpack_unpacker_buffer_capacity ptr + +foreign import ccall "msgpack_unpacker_buffer_capacity_wrap" msgpack_unpacker_buffer_capacity :: + Ptr () -> IO CSize + +-- | 'unpackerBufferConsumed' @up size@ notices that writed @size@ bytes to buffer. +unpackerBufferConsumed :: Unpacker -> Int -> IO () +unpackerBufferConsumed up size = + withForeignPtr up $ \ptr -> + msgpack_unpacker_buffer_consumed ptr (fromIntegral size) + +foreign import ccall "msgpack_unpacker_buffer_consumed_wrap" msgpack_unpacker_buffer_consumed :: + Ptr () -> CSize -> IO () + +-- | Write byte sequence to Unpacker. It is utility funciton, calls 'unpackerReserveBuffer', 'unpackerBuffer' and 'unpackerBufferConsumed'. +unpackerFeed :: Unpacker -> ByteString -> IO () +unpackerFeed up bs = + BS.useAsCStringLen bs $ \(str, len) -> do + True <- unpackerReserveBuffer up len + ptr <- unpackerBuffer up + copyArray ptr str len + unpackerBufferConsumed up len + +-- | Execute deserializing. It returns 0 when buffer contains not enough bytes, returns 1 when succeeded, returns negative value when it failed. +unpackerExecute :: Unpacker -> IO Int +unpackerExecute up = + withForeignPtr up $ \ptr -> + liftM fromIntegral $ msgpack_unpacker_execute ptr + +foreign import ccall "msgpack_unpacker_execute" msgpack_unpacker_execute :: + Ptr () -> IO CInt + +-- | Returns a deserialized object when 'unpackerExecute' returned 1. +unpackerData :: Unpacker -> IO Object +unpackerData up = + withForeignPtr up $ \ptr -> + allocaBytes (#size msgpack_object) $ \pobj -> do + msgpack_unpacker_data ptr pobj + peekObject pobj + +foreign import ccall "msgpack_unpacker_data_wrap" msgpack_unpacker_data :: + Ptr () -> Ptr () -> IO () + +-- | Release memory zone. The returned zone must be freed by calling 'freeZone'. +unpackerReleaseZone :: Unpacker -> IO Zone +unpackerReleaseZone up = + withForeignPtr up $ \ptr -> + msgpack_unpacker_release_zone ptr + +foreign import ccall "msgpack_unpacker_release_zone" msgpack_unpacker_release_zone :: + Ptr () -> IO (Ptr ()) + +-- | Free memory zone used by Unapcker. +unpackerResetZone :: Unpacker -> IO () +unpackerResetZone up = + withForeignPtr up $ \ptr -> + msgpack_unpacker_reset_zone ptr + +foreign import ccall "msgpack_unpacker_reset_zone" msgpack_unpacker_reset_zone :: + Ptr () -> IO () + +-- | Reset Unpacker state except memory zone. +unpackerReset :: Unpacker -> IO () +unpackerReset up = + withForeignPtr up $ \ptr -> + msgpack_unpacker_reset ptr + +foreign import ccall "msgpack_unpacker_reset" msgpack_unpacker_reset :: + Ptr () -> IO () + +-- | Returns number of bytes of sequence of deserializing object. +unpackerMessageSize :: Unpacker -> IO Int +unpackerMessageSize up = + withForeignPtr up $ \ptr -> + liftM fromIntegral $ msgpack_unpacker_message_size ptr + +foreign import ccall "msgpack_unpacker_message_size_wrap" msgpack_unpacker_message_size :: + Ptr () -> IO CSize + +type Zone = Ptr () + +-- | Create a new memory zone. It must be freed manually. +newZone :: IO Zone +newZone = + msgpack_zone_new (#const MSGPACK_ZONE_CHUNK_SIZE) + +-- | Free a memory zone. +freeZone :: Zone -> IO () +freeZone z = + msgpack_zone_free z + +-- | Create a memory zone, then execute argument, then free memory zone. +withZone :: (Zone -> IO a) -> IO a +withZone z = + bracket newZone freeZone z + +foreign import ccall "msgpack_zone_new" msgpack_zone_new :: + CSize -> IO Zone + +foreign import ccall "msgpack_zone_free" msgpack_zone_free :: + Zone -> IO () + +-- | Object Representation of MessagePack data. +data Object = + ObjectNil + | ObjectBool Bool + | ObjectInteger Int + | ObjectDouble Double + | ObjectRAW ByteString + | ObjectArray [Object] + | ObjectMap [(Object, Object)] + deriving (Show) + +peekObject :: Ptr a -> IO Object +peekObject ptr = do + typ <- (#peek msgpack_object, type) ptr + case (typ :: CInt) of + (#const MSGPACK_OBJECT_NIL) -> + return ObjectNil + (#const MSGPACK_OBJECT_BOOLEAN) -> + peekObjectBool ptr + (#const MSGPACK_OBJECT_POSITIVE_INTEGER) -> + peekObjectPositiveInteger ptr + (#const MSGPACK_OBJECT_NEGATIVE_INTEGER) -> + peekObjectNegativeInteger ptr + (#const MSGPACK_OBJECT_DOUBLE) -> + peekObjectDouble ptr + (#const MSGPACK_OBJECT_RAW) -> + peekObjectRAW ptr + (#const MSGPACK_OBJECT_ARRAY) -> + peekObjectArray ptr + (#const MSGPACK_OBJECT_MAP) -> + peekObjectMap ptr + _ -> + fail "peekObject: unknown object type" + +peekObjectBool :: Ptr a -> IO Object +peekObjectBool ptr = do + b <- (#peek msgpack_object, via.boolean) ptr + return $ ObjectBool $ (b :: CUChar) /= 0 + +peekObjectPositiveInteger :: Ptr a -> IO Object +peekObjectPositiveInteger ptr = do + n <- (#peek msgpack_object, via.u64) ptr + return $ ObjectInteger $ fromIntegral (n :: Word64) + +peekObjectNegativeInteger :: Ptr a -> IO Object +peekObjectNegativeInteger ptr = do + n <- (#peek msgpack_object, via.i64) ptr + return $ ObjectInteger $ fromIntegral (n :: Int64) + +peekObjectDouble :: Ptr a -> IO Object +peekObjectDouble ptr = do + d <- (#peek msgpack_object, via.dec) ptr + return $ ObjectDouble $ realToFrac (d :: CDouble) + +peekObjectRAW :: Ptr a -> IO Object +peekObjectRAW ptr = do + size <- (#peek msgpack_object, via.raw.size) ptr + p <- (#peek msgpack_object, via.raw.ptr) ptr + bs <- BS.packCStringLen (p, fromIntegral (size :: Word32)) + return $ ObjectRAW bs + +peekObjectArray :: Ptr a -> IO Object +peekObjectArray ptr = do + size <- (#peek msgpack_object, via.array.size) ptr + p <- (#peek msgpack_object, via.array.ptr) ptr + objs <- mapM (\i -> peekObject $ p `plusPtr` + ((#size msgpack_object) * i)) + [0..size-1] + return $ ObjectArray objs + +peekObjectMap :: Ptr a -> IO Object +peekObjectMap ptr = do + size <- (#peek msgpack_object, via.map.size) ptr + p <- (#peek msgpack_object, via.map.ptr) ptr + dat <- mapM (\i -> peekObjectKV $ p `plusPtr` + ((#size msgpack_object_kv) * i)) + [0..size-1] + return $ ObjectMap dat + +peekObjectKV :: Ptr a -> IO (Object, Object) +peekObjectKV ptr = do + k <- peekObject $ ptr `plusPtr` (#offset msgpack_object_kv, key) + v <- peekObject $ ptr `plusPtr` (#offset msgpack_object_kv, val) + return (k, v) + +-- | Pack a Object. +packObject :: Packer -> Object -> IO () +packObject pc ObjectNil = packNil pc >> return () + +packObject pc (ObjectBool b) = packBool pc b >> return () + +packObject pc (ObjectInteger n) = packInt pc n >> return () + +packObject pc (ObjectDouble d) = packDouble pc d >> return () + +packObject pc (ObjectRAW bs) = packRAW' pc bs >> return () + +packObject pc (ObjectArray ls) = do + packArray pc (length ls) + mapM_ (packObject pc) ls + +packObject pc (ObjectMap ls) = do + packMap pc (length ls) + mapM_ (\(a, b) -> packObject pc a >> packObject pc b) ls + +data UnpackReturn = + UnpackContinue -- ^ not enough bytes to unpack object + | UnpackParseError -- ^ got invalid bytes + | UnpackError -- ^ other error + deriving (Eq, Show) + +-- | Unpack a single MessagePack object from byte sequence. +unpackObject :: Zone -> ByteString -> IO (Either UnpackReturn (Int, Object)) +unpackObject z dat = + allocaBytes (#size msgpack_object) $ \ptr -> + BS.useAsCStringLen dat $ \(str, len) -> + alloca $ \poff -> do + ret <- msgpack_unpack str (fromIntegral len) poff z ptr + case ret of + (#const MSGPACK_UNPACK_SUCCESS) -> do + off <- peek poff + obj <- peekObject ptr + return $ Right (fromIntegral off, obj) + (#const MSGPACK_UNPACK_EXTRA_BYTES) -> do + off <- peek poff + obj <- peekObject ptr + return $ Right (fromIntegral off, obj) + (#const MSGPACK_UNPACK_CONTINUE) -> + return $ Left UnpackContinue + (#const MSGPACK_UNPACK_PARSE_ERROR) -> + return $ Left UnpackParseError + _ -> + return $ Left UnpackError + +foreign import ccall "msgpack_unpack" msgpack_unpack :: + Ptr CChar -> CSize -> Ptr CSize -> Zone -> Ptr () -> IO CInt diff --git a/haskell/src/Data/MessagePack/Class.hs b/haskell/src/Data/MessagePack/Class.hs new file mode 100644 index 0000000..f50a4d8 --- /dev/null +++ b/haskell/src/Data/MessagePack/Class.hs @@ -0,0 +1,97 @@ +{-# LANGUAGE TypeSynonymInstances #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE OverlappingInstances #-} +{-# LANGUAGE IncoherentInstances #-} + +-------------------------------------------------------------------- +-- | +-- Module : Data.MessagePack.Class +-- Copyright : (c) Hideyuki Tanaka, 2009 +-- License : BSD3 +-- +-- Maintainer: tanaka.hideyuki@gmail.com +-- Stability : experimental +-- Portability: portable +-- +-- Serializing Haskell values to and from MessagePack Objects. +-- +-------------------------------------------------------------------- + +module Data.MessagePack.Class( + -- * Serialization to and from Object + OBJECT(..), + Result, + pack, + ) where + +import Control.Monad.Error +import Data.ByteString.Char8 (ByteString) +import qualified Data.ByteString.Char8 as C8 +import Data.Either + +import Data.MessagePack.Base + +-- | The class of types serializable to and from MessagePack object +class OBJECT a where + toObject :: a -> Object + fromObject :: Object -> Result a + +-- | A type for parser results +type Result a = Either String a + +instance OBJECT Object where + toObject = id + fromObject = Right + +fromObjectError :: String +fromObjectError = "fromObject: cannot cast" + +instance OBJECT Int where + toObject = ObjectInteger + fromObject (ObjectInteger n) = Right n + fromObject _ = Left fromObjectError + +instance OBJECT Bool where + toObject = ObjectBool + fromObject (ObjectBool b) = Right b + fromObject _ = Left fromObjectError + +instance OBJECT Double where + toObject = ObjectDouble + fromObject (ObjectDouble d) = Right d + fromObject _ = Left fromObjectError + +instance OBJECT ByteString where + toObject = ObjectRAW + fromObject (ObjectRAW bs) = Right bs + fromObject _ = Left fromObjectError + +instance OBJECT String where + toObject = toObject . C8.pack + fromObject obj = liftM C8.unpack $ fromObject obj + +instance OBJECT a => OBJECT [a] where + toObject = ObjectArray . map toObject + fromObject (ObjectArray arr) = + mapM fromObject arr + fromObject _ = + Left fromObjectError + +instance (OBJECT a, OBJECT b) => OBJECT [(a, b)] where + toObject = + ObjectMap . map (\(a, b) -> (toObject a, toObject b)) + fromObject (ObjectMap mem) = do + mapM (\(a, b) -> liftM2 (,) (fromObject a) (fromObject b)) mem + fromObject _ = + Left fromObjectError + +instance OBJECT a => OBJECT (Maybe a) where + toObject (Just a) = toObject a + toObject Nothing = ObjectNil + + fromObject ObjectNil = return Nothing + fromObject obj = liftM Just $ fromObject obj + +-- | Pack a serializable Haskell value. +pack :: OBJECT a => Packer -> a -> IO () +pack pc = packObject pc . toObject diff --git a/haskell/src/Data/MessagePack/Feed.hs b/haskell/src/Data/MessagePack/Feed.hs new file mode 100644 index 0000000..afd3f6c --- /dev/null +++ b/haskell/src/Data/MessagePack/Feed.hs @@ -0,0 +1,59 @@ +-------------------------------------------------------------------- +-- | +-- Module : Data.MessagePack.Feed +-- Copyright : (c) Hideyuki Tanaka, 2009 +-- License : BSD3 +-- +-- Maintainer: tanaka.hideyuki@gmail.com +-- Stability : experimental +-- Portability: portable +-- +-- Feeders for Stream Deserializers +-- +-------------------------------------------------------------------- + +module Data.MessagePack.Feed( + -- * Feeder type + Feeder, + -- * Feeders + feederFromHandle, + feederFromFile, + feederFromString, + ) where + +import Control.Monad +import Data.ByteString (ByteString) +import qualified Data.ByteString as BS +import Data.IORef +import System.IO + +-- | Feeder returns Just ByteString when bytes remains, otherwise Nothing. +type Feeder = IO (Maybe ByteString) + +-- | Feeder from Handle +feederFromHandle :: Handle -> IO Feeder +feederFromHandle h = return $ do + bs <- BS.hGet h bufSize + if BS.length bs > 0 + then return $ Just bs + else do + hClose h + return Nothing + where + bufSize = 4096 + +-- | Feeder from File +feederFromFile :: FilePath -> IO Feeder +feederFromFile path = + openFile path ReadMode >>= feederFromHandle + +-- | Feeder from ByteString +feederFromString :: ByteString -> IO Feeder +feederFromString bs = do + r <- newIORef (Just bs) + return $ f r + where + f r = do + mb <- readIORef r + writeIORef r Nothing + return mb diff --git a/haskell/src/Data/MessagePack/Monad.hs b/haskell/src/Data/MessagePack/Monad.hs new file mode 100644 index 0000000..bf1514f --- /dev/null +++ b/haskell/src/Data/MessagePack/Monad.hs @@ -0,0 +1,153 @@ +-------------------------------------------------------------------- +-- | +-- Module : Data.MessagePack.Monad +-- Copyright : (c) Hideyuki Tanaka, 2009 +-- License : BSD3 +-- +-- Maintainer: tanaka.hideyuki@gmail.com +-- Stability : experimental +-- Portability: portable +-- +-- Monadic Stream Serializers and Deserializers +-- +-------------------------------------------------------------------- + +module Data.MessagePack.Monad( + -- * Classes + MonadPacker(..), + MonadUnpacker(..), + + -- * Packer and Unpacker type + PackerT(..), + UnpackerT(..), + + -- * Packers + packToString, + packToHandle, + packToFile, + + -- * Unpackers + unpackFrom, + unpackFromString, + unpackFromHandle, + unpackFromFile, + ) where + +import Control.Monad +import Control.Monad.Trans +import Data.ByteString (ByteString) +import qualified Data.ByteString as BS +import System.IO + +import Data.MessagePack.Base hiding (Unpacker) +import qualified Data.MessagePack.Base as Base +import Data.MessagePack.Class +import Data.MessagePack.Feed + +class Monad m => MonadPacker m where + -- | Serialize a object + put :: OBJECT a => a -> m () + +class Monad m => MonadUnpacker m where + -- | Deserialize a object + get :: OBJECT a => m a + +-- | Serializer Type +newtype PackerT m r = PackerT { runPackerT :: Base.Packer -> m r } + +instance Monad m => Monad (PackerT m) where + a >>= b = + PackerT $ \pc -> do + r <- runPackerT a pc + runPackerT (b r) pc + + return r = + PackerT $ \_ -> return r + +instance MonadTrans PackerT where + lift m = PackerT $ \_ -> m + +instance MonadIO m => MonadIO (PackerT m) where + liftIO = lift . liftIO + +instance MonadIO m => MonadPacker (PackerT m) where + put v = PackerT $ \pc -> liftIO $ do + pack pc v + +-- | Execute given serializer and returns byte sequence. +packToString :: MonadIO m => PackerT m r -> m ByteString +packToString m = do + sb <- liftIO $ newSimpleBuffer + pc <- liftIO $ newPacker sb + runPackerT m pc + liftIO $ simpleBufferData sb + +-- | Execcute given serializer and write byte sequence to Handle. +packToHandle :: MonadIO m => Handle -> PackerT m r -> m () +packToHandle h m = do + sb <- packToString m + liftIO $ BS.hPut h sb + liftIO $ hFlush h + +-- | Execute given serializer and write byte sequence to file. +packToFile :: MonadIO m => FilePath -> PackerT m r -> m () +packToFile p m = do + sb <- packToString m + liftIO $ BS.writeFile p sb + +-- | Deserializer type +newtype UnpackerT m r = UnpackerT { runUnpackerT :: Base.Unpacker -> Feeder -> m r } + +instance Monad m => Monad (UnpackerT m) where + a >>= b = + UnpackerT $ \up feed -> do + r <- runUnpackerT a up feed + runUnpackerT (b r) up feed + + return r = + UnpackerT $ \_ _ -> return r + +instance MonadTrans UnpackerT where + lift m = UnpackerT $ \_ _ -> m + +instance MonadIO m => MonadIO (UnpackerT m) where + liftIO = lift . liftIO + +instance MonadIO m => MonadUnpacker (UnpackerT m) where + get = UnpackerT $ \up feed -> liftIO $ do + resp <- unpackerExecute up + guard $ resp>=0 + when (resp==0) $ do + Just bs <- feed + unpackerFeed up bs + resp2 <- unpackerExecute up + guard $ resp2==1 + obj <- unpackerData up + freeZone =<< unpackerReleaseZone up + unpackerReset up + let Right r = fromObject obj + return r + +-- | Execute deserializer using given feeder. +unpackFrom :: MonadIO m => Feeder -> UnpackerT m r -> m r +unpackFrom f m = do + up <- liftIO $ newUnpacker defaultInitialBufferSize + runUnpackerT m up f + +-- | Execute deserializer using given handle. +unpackFromHandle :: MonadIO m => Handle -> UnpackerT m r -> m r +unpackFromHandle h m = + flip unpackFrom m =<< liftIO (feederFromHandle h) + +-- | Execute deserializer using given file content. +unpackFromFile :: MonadIO m => FilePath -> UnpackerT m r -> m r +unpackFromFile p m = do + h <- liftIO $ openFile p ReadMode + r <- flip unpackFrom m =<< liftIO (feederFromHandle h) + liftIO $ hClose h + return r + +-- | Execute deserializer from given byte sequence. +unpackFromString :: MonadIO m => ByteString -> UnpackerT m r -> m r +unpackFromString bs m = do + flip unpackFrom m =<< liftIO (feederFromString bs) diff --git a/haskell/src/Data/MessagePack/Stream.hs b/haskell/src/Data/MessagePack/Stream.hs new file mode 100644 index 0000000..bd17f46 --- /dev/null +++ b/haskell/src/Data/MessagePack/Stream.hs @@ -0,0 +1,84 @@ +-------------------------------------------------------------------- +-- | +-- Module : Data.MessagePack.Stream +-- Copyright : (c) Hideyuki Tanaka, 2009 +-- License : BSD3 +-- +-- Maintainer: tanaka.hideyuki@gmail.com +-- Stability : experimental +-- Portability: portable +-- +-- Lazy Stream Serializers and Deserializers +-- +-------------------------------------------------------------------- + +module Data.MessagePack.Stream( + unpackObjects, + unpackObjectsFromFile, + unpackObjectsFromHandle, + unpackObjectsFromString, + ) where + +import Control.Monad +import Data.ByteString (ByteString) +import qualified Data.ByteString as BS +import System.IO +import System.IO.Unsafe + +import Data.MessagePack.Base +import Data.MessagePack.Feed + +-- | Unpack objects using given feeder. +unpackObjects :: Feeder -> IO [Object] +unpackObjects feeder = do + up <- newUnpacker defaultInitialBufferSize + f up + where + f up = unsafeInterleaveIO $ do + mbo <- unpackOnce up + case mbo of + Just o -> do + os <- f up + return $ o:os + Nothing -> + return [] + + unpackOnce up = do + resp <- unpackerExecute up + case resp of + 0 -> do + r <- feedOnce up + if r + then unpackOnce up + else return Nothing + 1 -> do + obj <- unpackerData up + freeZone =<< unpackerReleaseZone up + unpackerReset up + return $ Just obj + _ -> + error $ "unpackerExecute fails: " ++ show resp + + feedOnce up = do + dat <- feeder + case dat of + Nothing -> + return False + Just bs -> do + unpackerFeed up bs + return True + +-- | Unpack objects from file. +unpackObjectsFromFile :: FilePath -> IO [Object] +unpackObjectsFromFile fname = + unpackObjects =<< feederFromFile fname + +-- | Unpack objects from handle. +unpackObjectsFromHandle :: Handle -> IO [Object] +unpackObjectsFromHandle h = + unpackObjects =<< feederFromHandle h + +-- | Unpack oobjects from given byte sequence. +unpackObjectsFromString :: ByteString -> IO [Object] +unpackObjectsFromString bs = + unpackObjects =<< feederFromString bs diff --git a/haskell/test/Monad.hs b/haskell/test/Monad.hs new file mode 100644 index 0000000..4bee5c5 --- /dev/null +++ b/haskell/test/Monad.hs @@ -0,0 +1,16 @@ +import Control.Monad.Trans +import Data.MessagePack + +main = do + sb <- packToString $ do + put [1,2,3::Int] + put (3.14 :: Double) + put "Hoge" + + print sb + + unpackFromString sb $ do + arr <- get + dbl <- get + str <- get + liftIO $ print (arr :: [Int], dbl :: Double, str :: String) diff --git a/haskell/test/Stream.hs b/haskell/test/Stream.hs new file mode 100644 index 0000000..ce060de --- /dev/null +++ b/haskell/test/Stream.hs @@ -0,0 +1,14 @@ +import Control.Applicative +import qualified Data.ByteString as BS +import Data.MessagePack + +main = do + sb <- newSimpleBuffer + pc <- newPacker sb + pack pc [1,2,3::Int] + pack pc True + pack pc "hoge" + bs <- simpleBufferData sb + + os <- unpackObjectsFromString bs + mapM_ print os diff --git a/haskell/test/Test.hs b/haskell/test/Test.hs new file mode 100644 index 0000000..4e713ba --- /dev/null +++ b/haskell/test/Test.hs @@ -0,0 +1,36 @@ +import Control.Monad +import Data.MessagePack + +{- +main = do + sb <- newSimpleBuffer + pc <- newPacker sb + + pack pc [(1,2),(2,3),(3::Int,4::Int)] + pack pc [4,5,6::Int] + pack pc "hoge" + + bs <- simpleBufferData sb + print bs + + up <- newUnpacker defaultInitialBufferSize + + unpackerFeed up bs + + let f = do + res <- unpackerExecute up + when (res==1) $ do + obj <- unpackerData up + print obj + f + + f + + return () +-} + +main = do + bs <- packb [(1,2),(2,3),(3::Int,4::Int)] + print bs + dat <- unpackb bs + print (dat :: Result [(Int, Int)]) From 11f7aa4212a2539b008f7e357474ebc412647c71 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sun, 18 Apr 2010 15:11:50 +0900 Subject: [PATCH 0413/1172] cpp: fixed small typo in configure.in --- cpp/configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/configure.in b/cpp/configure.in index a4cb4d8..8c59dda 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -33,7 +33,7 @@ if test "$msgpack_cv_atomic_ops" != "yes"; then Note that gcc < 4.1 is not supported. If you are using gcc >= 4.1 and the default target CPU architecture is "i386", try to -add CFLAGS="--march=i686" and CXXFLAGS="-march=i668" options to ./configure as follows: +add CFLAGS="--march=i686" and CXXFLAGS="-march=i686" options to ./configure as follows: $ ./configure CFLAGS="-march=i686" CXXFLAGS="-march=i686" ]) From 20fe9b6dde977cb7d923167292bbe89027a01795 Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Mon, 19 Apr 2010 22:39:53 +0900 Subject: [PATCH 0414/1172] added pom.xml --- java/.gitignore | 4 + java/.settings/org.eclipse.jdt.core.prefs | 5 + java/Makefile | 16 +++ java/pom.xml | 125 ++++++++++++++++++++++ 4 files changed, 150 insertions(+) create mode 100755 java/.gitignore create mode 100755 java/.settings/org.eclipse.jdt.core.prefs create mode 100755 java/Makefile create mode 100755 java/pom.xml diff --git a/java/.gitignore b/java/.gitignore new file mode 100755 index 0000000..156d227 --- /dev/null +++ b/java/.gitignore @@ -0,0 +1,4 @@ +target +.project +.classpath +*~ diff --git a/java/.settings/org.eclipse.jdt.core.prefs b/java/.settings/org.eclipse.jdt.core.prefs new file mode 100755 index 0000000..c75ad47 --- /dev/null +++ b/java/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +#Mon Apr 19 22:18:48 JST 2010 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.source=1.6 +org.eclipse.jdt.core.compiler.compliance=1.6 diff --git a/java/Makefile b/java/Makefile new file mode 100755 index 0000000..2b6b07d --- /dev/null +++ b/java/Makefile @@ -0,0 +1,16 @@ + +.PHONY: compile test eclipse clean + + +compile: + mvn compile + +test: + mvn test + +# generate .project and .classpath file for Eclipse +eclipse: + mvn eclipse:eclipse + +clean: + mvn clean diff --git a/java/pom.xml b/java/pom.xml new file mode 100755 index 0000000..5a7b4a5 --- /dev/null +++ b/java/pom.xml @@ -0,0 +1,125 @@ + + + 4.0.0 + org.msgpack + msgpack + MessagePack for Java + 0.0.1-SNAPSHOT + MessagePack for Java + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + scm:git://github.com/msgpack/msgpack.git + + + + src + test + + + + src/main/resources + + + + + src/test/resources + + + + + + maven-compiler-plugin + + 1.6 + 1.6 + + + + + maven-eclipse-plugin + 2.5.1 + + + + maven-release-plugin + + + deploy + scm:git://github.com/msgpack/msgpack.git + + + + + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + ${project.name} ${project.version} API + true + en_US + UTF-8 + + + + + org.apache.maven.plugins + maven-jxr-plugin + + true + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + + true + + + + + + + + + sourceforge + + + sourceforge.net + Repository at sourceforge.net + scpexe://shell.sourceforge.net/home/groups/m/ms/msgpack/htdocs/maven2/ + + + + + + + + junit + junit + 4.8.1 + test + + + + org.apache.activemq.protobuf + activemq-protobuf + 1.0 + test + + + + From f2622e54e350bcec9233084a0d52eb61dcdeddb1 Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Mon, 19 Apr 2010 23:05:00 +0900 Subject: [PATCH 0415/1172] moved src and test codes to src/{main,test}/java --- java/src/{ => main/java}/org/msgpack/MessageMergeable.java | 0 java/src/{ => main/java}/org/msgpack/MessagePackable.java | 0 java/src/{ => main/java}/org/msgpack/MessageTypeException.java | 0 java/src/{ => main/java}/org/msgpack/Packer.java | 0 java/src/{ => main/java}/org/msgpack/Schema.java | 0 java/src/{ => main/java}/org/msgpack/UnbufferedUnpacker.java | 0 java/src/{ => main/java}/org/msgpack/UnpackException.java | 0 java/src/{ => main/java}/org/msgpack/UnpackIterator.java | 0 java/src/{ => main/java}/org/msgpack/Unpacker.java | 0 java/src/{ => main/java}/org/msgpack/impl/UnpackerImpl.java | 0 java/src/{ => main/java}/org/msgpack/schema/ArraySchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/ByteSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/ClassGenerator.java | 0 java/src/{ => main/java}/org/msgpack/schema/ClassSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/DoubleSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/FieldSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/FloatSchema.java | 0 .../{ => main/java}/org/msgpack/schema/GenericClassSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/GenericSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/IArraySchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/IMapSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/IntSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/LongSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/MapSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/RawSchema.java | 0 .../{ => main/java}/org/msgpack/schema/ReflectionClassSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/SSchemaParser.java | 0 java/src/{ => main/java}/org/msgpack/schema/ShortSchema.java | 0 .../{ => main/java}/org/msgpack/schema/SpecificClassSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/StringSchema.java | 0 java/{test => src/test/java}/org/msgpack/TestPackUnpack.java | 0 java/{test => src/test/java}/org/msgpack/TestSample.java | 0 32 files changed, 0 insertions(+), 0 deletions(-) rename java/src/{ => main/java}/org/msgpack/MessageMergeable.java (100%) rename java/src/{ => main/java}/org/msgpack/MessagePackable.java (100%) rename java/src/{ => main/java}/org/msgpack/MessageTypeException.java (100%) rename java/src/{ => main/java}/org/msgpack/Packer.java (100%) rename java/src/{ => main/java}/org/msgpack/Schema.java (100%) rename java/src/{ => main/java}/org/msgpack/UnbufferedUnpacker.java (100%) rename java/src/{ => main/java}/org/msgpack/UnpackException.java (100%) rename java/src/{ => main/java}/org/msgpack/UnpackIterator.java (100%) rename java/src/{ => main/java}/org/msgpack/Unpacker.java (100%) rename java/src/{ => main/java}/org/msgpack/impl/UnpackerImpl.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/ArraySchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/ByteSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/ClassGenerator.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/ClassSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/DoubleSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/FieldSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/FloatSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/GenericClassSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/GenericSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/IArraySchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/IMapSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/IntSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/LongSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/MapSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/RawSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/ReflectionClassSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/SSchemaParser.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/ShortSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/SpecificClassSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/StringSchema.java (100%) rename java/{test => src/test/java}/org/msgpack/TestPackUnpack.java (100%) rename java/{test => src/test/java}/org/msgpack/TestSample.java (100%) diff --git a/java/src/org/msgpack/MessageMergeable.java b/java/src/main/java/org/msgpack/MessageMergeable.java similarity index 100% rename from java/src/org/msgpack/MessageMergeable.java rename to java/src/main/java/org/msgpack/MessageMergeable.java diff --git a/java/src/org/msgpack/MessagePackable.java b/java/src/main/java/org/msgpack/MessagePackable.java similarity index 100% rename from java/src/org/msgpack/MessagePackable.java rename to java/src/main/java/org/msgpack/MessagePackable.java diff --git a/java/src/org/msgpack/MessageTypeException.java b/java/src/main/java/org/msgpack/MessageTypeException.java similarity index 100% rename from java/src/org/msgpack/MessageTypeException.java rename to java/src/main/java/org/msgpack/MessageTypeException.java diff --git a/java/src/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java similarity index 100% rename from java/src/org/msgpack/Packer.java rename to java/src/main/java/org/msgpack/Packer.java diff --git a/java/src/org/msgpack/Schema.java b/java/src/main/java/org/msgpack/Schema.java similarity index 100% rename from java/src/org/msgpack/Schema.java rename to java/src/main/java/org/msgpack/Schema.java diff --git a/java/src/org/msgpack/UnbufferedUnpacker.java b/java/src/main/java/org/msgpack/UnbufferedUnpacker.java similarity index 100% rename from java/src/org/msgpack/UnbufferedUnpacker.java rename to java/src/main/java/org/msgpack/UnbufferedUnpacker.java diff --git a/java/src/org/msgpack/UnpackException.java b/java/src/main/java/org/msgpack/UnpackException.java similarity index 100% rename from java/src/org/msgpack/UnpackException.java rename to java/src/main/java/org/msgpack/UnpackException.java diff --git a/java/src/org/msgpack/UnpackIterator.java b/java/src/main/java/org/msgpack/UnpackIterator.java similarity index 100% rename from java/src/org/msgpack/UnpackIterator.java rename to java/src/main/java/org/msgpack/UnpackIterator.java diff --git a/java/src/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java similarity index 100% rename from java/src/org/msgpack/Unpacker.java rename to java/src/main/java/org/msgpack/Unpacker.java diff --git a/java/src/org/msgpack/impl/UnpackerImpl.java b/java/src/main/java/org/msgpack/impl/UnpackerImpl.java similarity index 100% rename from java/src/org/msgpack/impl/UnpackerImpl.java rename to java/src/main/java/org/msgpack/impl/UnpackerImpl.java diff --git a/java/src/org/msgpack/schema/ArraySchema.java b/java/src/main/java/org/msgpack/schema/ArraySchema.java similarity index 100% rename from java/src/org/msgpack/schema/ArraySchema.java rename to java/src/main/java/org/msgpack/schema/ArraySchema.java diff --git a/java/src/org/msgpack/schema/ByteSchema.java b/java/src/main/java/org/msgpack/schema/ByteSchema.java similarity index 100% rename from java/src/org/msgpack/schema/ByteSchema.java rename to java/src/main/java/org/msgpack/schema/ByteSchema.java diff --git a/java/src/org/msgpack/schema/ClassGenerator.java b/java/src/main/java/org/msgpack/schema/ClassGenerator.java similarity index 100% rename from java/src/org/msgpack/schema/ClassGenerator.java rename to java/src/main/java/org/msgpack/schema/ClassGenerator.java diff --git a/java/src/org/msgpack/schema/ClassSchema.java b/java/src/main/java/org/msgpack/schema/ClassSchema.java similarity index 100% rename from java/src/org/msgpack/schema/ClassSchema.java rename to java/src/main/java/org/msgpack/schema/ClassSchema.java diff --git a/java/src/org/msgpack/schema/DoubleSchema.java b/java/src/main/java/org/msgpack/schema/DoubleSchema.java similarity index 100% rename from java/src/org/msgpack/schema/DoubleSchema.java rename to java/src/main/java/org/msgpack/schema/DoubleSchema.java diff --git a/java/src/org/msgpack/schema/FieldSchema.java b/java/src/main/java/org/msgpack/schema/FieldSchema.java similarity index 100% rename from java/src/org/msgpack/schema/FieldSchema.java rename to java/src/main/java/org/msgpack/schema/FieldSchema.java diff --git a/java/src/org/msgpack/schema/FloatSchema.java b/java/src/main/java/org/msgpack/schema/FloatSchema.java similarity index 100% rename from java/src/org/msgpack/schema/FloatSchema.java rename to java/src/main/java/org/msgpack/schema/FloatSchema.java diff --git a/java/src/org/msgpack/schema/GenericClassSchema.java b/java/src/main/java/org/msgpack/schema/GenericClassSchema.java similarity index 100% rename from java/src/org/msgpack/schema/GenericClassSchema.java rename to java/src/main/java/org/msgpack/schema/GenericClassSchema.java diff --git a/java/src/org/msgpack/schema/GenericSchema.java b/java/src/main/java/org/msgpack/schema/GenericSchema.java similarity index 100% rename from java/src/org/msgpack/schema/GenericSchema.java rename to java/src/main/java/org/msgpack/schema/GenericSchema.java diff --git a/java/src/org/msgpack/schema/IArraySchema.java b/java/src/main/java/org/msgpack/schema/IArraySchema.java similarity index 100% rename from java/src/org/msgpack/schema/IArraySchema.java rename to java/src/main/java/org/msgpack/schema/IArraySchema.java diff --git a/java/src/org/msgpack/schema/IMapSchema.java b/java/src/main/java/org/msgpack/schema/IMapSchema.java similarity index 100% rename from java/src/org/msgpack/schema/IMapSchema.java rename to java/src/main/java/org/msgpack/schema/IMapSchema.java diff --git a/java/src/org/msgpack/schema/IntSchema.java b/java/src/main/java/org/msgpack/schema/IntSchema.java similarity index 100% rename from java/src/org/msgpack/schema/IntSchema.java rename to java/src/main/java/org/msgpack/schema/IntSchema.java diff --git a/java/src/org/msgpack/schema/LongSchema.java b/java/src/main/java/org/msgpack/schema/LongSchema.java similarity index 100% rename from java/src/org/msgpack/schema/LongSchema.java rename to java/src/main/java/org/msgpack/schema/LongSchema.java diff --git a/java/src/org/msgpack/schema/MapSchema.java b/java/src/main/java/org/msgpack/schema/MapSchema.java similarity index 100% rename from java/src/org/msgpack/schema/MapSchema.java rename to java/src/main/java/org/msgpack/schema/MapSchema.java diff --git a/java/src/org/msgpack/schema/RawSchema.java b/java/src/main/java/org/msgpack/schema/RawSchema.java similarity index 100% rename from java/src/org/msgpack/schema/RawSchema.java rename to java/src/main/java/org/msgpack/schema/RawSchema.java diff --git a/java/src/org/msgpack/schema/ReflectionClassSchema.java b/java/src/main/java/org/msgpack/schema/ReflectionClassSchema.java similarity index 100% rename from java/src/org/msgpack/schema/ReflectionClassSchema.java rename to java/src/main/java/org/msgpack/schema/ReflectionClassSchema.java diff --git a/java/src/org/msgpack/schema/SSchemaParser.java b/java/src/main/java/org/msgpack/schema/SSchemaParser.java similarity index 100% rename from java/src/org/msgpack/schema/SSchemaParser.java rename to java/src/main/java/org/msgpack/schema/SSchemaParser.java diff --git a/java/src/org/msgpack/schema/ShortSchema.java b/java/src/main/java/org/msgpack/schema/ShortSchema.java similarity index 100% rename from java/src/org/msgpack/schema/ShortSchema.java rename to java/src/main/java/org/msgpack/schema/ShortSchema.java diff --git a/java/src/org/msgpack/schema/SpecificClassSchema.java b/java/src/main/java/org/msgpack/schema/SpecificClassSchema.java similarity index 100% rename from java/src/org/msgpack/schema/SpecificClassSchema.java rename to java/src/main/java/org/msgpack/schema/SpecificClassSchema.java diff --git a/java/src/org/msgpack/schema/StringSchema.java b/java/src/main/java/org/msgpack/schema/StringSchema.java similarity index 100% rename from java/src/org/msgpack/schema/StringSchema.java rename to java/src/main/java/org/msgpack/schema/StringSchema.java diff --git a/java/test/org/msgpack/TestPackUnpack.java b/java/src/test/java/org/msgpack/TestPackUnpack.java similarity index 100% rename from java/test/org/msgpack/TestPackUnpack.java rename to java/src/test/java/org/msgpack/TestPackUnpack.java diff --git a/java/test/org/msgpack/TestSample.java b/java/src/test/java/org/msgpack/TestSample.java similarity index 100% rename from java/test/org/msgpack/TestSample.java rename to java/src/test/java/org/msgpack/TestSample.java From d693d9270225eb6e6ab133b124f1adc626ed319c Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Mon, 19 Apr 2010 23:34:18 +0900 Subject: [PATCH 0416/1172] removed unnecessary settings --- java/Makefile | 10 +++++++++- java/pom.xml | 18 +----------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/java/Makefile b/java/Makefile index 2b6b07d..7885a13 100755 --- a/java/Makefile +++ b/java/Makefile @@ -1,6 +1,14 @@ -.PHONY: compile test eclipse clean +.PHONY: compile test eclipse clean package +all: + compile + +package: + mvn package + +install: + mvn install compile: mvn compile diff --git a/java/pom.xml b/java/pom.xml index 5a7b4a5..321e8d5 100755 --- a/java/pom.xml +++ b/java/pom.xml @@ -4,7 +4,7 @@ org.msgpack msgpack MessagePack for Java - 0.0.1-SNAPSHOT + 1.0-SNAPSHOT MessagePack for Java @@ -20,9 +20,6 @@ - src - test - src/main/resources @@ -77,17 +74,11 @@ org.apache.maven.plugins maven-jxr-plugin - - true - org.apache.maven.plugins maven-surefire-report-plugin - - true - @@ -114,12 +105,5 @@ test - - org.apache.activemq.protobuf - activemq-protobuf - 1.0 - test - - From 5aa620966482ead63198ff15f59aad482cacb180 Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Mon, 19 Apr 2010 23:52:40 +0900 Subject: [PATCH 0417/1172] add README --- java/README | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 java/README diff --git a/java/README b/java/README new file mode 100755 index 0000000..33e6376 --- /dev/null +++ b/java/README @@ -0,0 +1,28 @@ + +To build the JAR file of Message Pack, you need to install Maven (http://maven.apache.org), then type the following command: + +$ mvn package + +To locally install the project, type +$ mvn install + +To generate project files (.project, .classpath) for Eclipse, do + +$ mvn eclipse:eclipse + +then import the folder from your Eclipse. + +Next, open the preference page in Eclipse and add the CLASSPATH variable: + +M2_REPO = $HOME/.m2/repository + +where $HOME is your home directory. In Windows XP, $HOME is: +C:/Documents and Settings/(user name)/.m2/repository + + +# How to release the project (compile, test, tagging, deploy) + +$ mvn release:prepare +$ mvn release:perform + + From bccac610a462f6f989f10d58b9fc55522624d76a Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Tue, 20 Apr 2010 00:05:26 +0900 Subject: [PATCH 0418/1172] changed the src/test folder --- java/build.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 java/build.xml diff --git a/java/build.xml b/java/build.xml old mode 100644 new mode 100755 index f6e93ae..bae8923 --- a/java/build.xml +++ b/java/build.xml @@ -11,7 +11,7 @@ - + @@ -24,11 +24,11 @@ - + - + From 354af69f6204ba4523cf7ca3e50f2af1ad8d609e Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 22 Apr 2010 14:38:10 +0900 Subject: [PATCH 0419/1172] ruby: fixes SEGV on MessagePack_Unpacker_each --- cpp/README.md | 4 +- ruby/msgpack_test.rb | 1 + ruby/unpack.c | 318 ++++++++++++++++++++----------------------- 3 files changed, 149 insertions(+), 174 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index cc1ba38..454ce1a 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -28,7 +28,7 @@ To use the library in your program, include msgpack.hpp header and link msgpack std::vector target; target.push_back("Hello,"); target.push_back("World!"); - + // Serialize it. msgpack::sbuffer buffer; // simple buffer msgpack::pack(&buffer, target); @@ -38,7 +38,7 @@ To use the library in your program, include msgpack.hpp header and link msgpack msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(buffer.data, buffer.size, NULL, &mempool, &obj); - + if(ret != msgapck::UNPACK_SUCCESS) { // error check exit(1); diff --git a/ruby/msgpack_test.rb b/ruby/msgpack_test.rb index 37db6a0..455aa40 100644 --- a/ruby/msgpack_test.rb +++ b/ruby/msgpack_test.rb @@ -209,6 +209,7 @@ class MessagePackTestFormat < Test::Unit::TestCase obj.to_msgpack.split(//).each do |b| pac.feed(b) pac.each {|o| + GC.start assert_equal(obj, o) parsed += 1 } diff --git a/ruby/unpack.c b/ruby/unpack.c index 11572c3..73aeeeb 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -31,7 +31,6 @@ typedef struct { int finished; VALUE source; size_t offset; - size_t parsed; VALUE buffer; VALUE stream; VALUE streambuf; @@ -144,9 +143,71 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha rb_raise(rb_eTypeError, "instance of String needed"); \ } + +static VALUE template_execute_rescue(VALUE nouse) +{ + rb_gc_enable(); +#ifdef RUBY_VM + rb_exc_raise(rb_errinfo()); +#else + rb_exc_raise(ruby_errinfo); +#endif +} + +static VALUE template_execute_do(VALUE argv) +{ + VALUE* args = (VALUE*)argv; + + msgpack_unpack_t* mp = (msgpack_unpack_t*)args[0]; + char* dptr = (char*)args[1]; + size_t dlen = (size_t)args[2]; + size_t* from = (size_t*)args[3]; + + int ret = template_execute(mp, dptr, dlen, from); + + return (VALUE)ret; +} + +static int template_execute_wrap(msgpack_unpack_t* mp, + VALUE str, size_t dlen, size_t* from) +{ + VALUE args[4] = { + (VALUE)mp, + (VALUE)RSTRING_PTR(str), + (VALUE)dlen, + (VALUE)from, + }; + +#ifdef HAVE_RUBY_ENCODING_H + // FIXME encodingã‚’ASCII-8BITã«ã™ã‚‹ + int enc_orig = rb_enc_get_index(str); + rb_enc_set_index(str, s_ascii_8bit); +#endif + + // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ + rb_gc_disable(); + + mp->user.source = str; + + int ret = (int)rb_rescue(template_execute_do, (VALUE)args, + template_execute_rescue, Qnil); + + mp->user.source = Qnil; + + rb_gc_enable(); + +#ifdef HAVE_RUBY_ENCODING_H + rb_enc_set_index(str, enc_orig); +#endif + + return ret; +} + + static VALUE cUnpacker; static VALUE eUnpackError; + static void MessagePack_Unpacker_free(void* data) { if(data) { free(data); } @@ -173,14 +234,6 @@ static VALUE MessagePack_Unpacker_alloc(VALUE klass) return obj; } -static VALUE MessagePack_Unpacker_reset(VALUE self) -{ - UNPACKER(self, mp); - template_init(mp); - mp->user.finished = 0; - return self; -} - static ID append_method_of(VALUE stream) { if(rb_respond_to(stream, s_sysread)) { @@ -204,10 +257,10 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) rb_raise(rb_eArgError, "wrong number of arguments (%d for 0)", argc); } - MessagePack_Unpacker_reset(self); UNPACKER(self, mp); + template_init(mp); + mp->user.finished = 0; mp->user.offset = 0; - mp->user.parsed = 0; mp->user.buffer = rb_str_new("",0); mp->user.stream = stream; mp->user.streambuf = rb_str_new("",0); @@ -215,102 +268,6 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) return self; } - -static VALUE MessagePack_Unpacker_execute_do(VALUE argv) -{ - VALUE* args = (VALUE*)argv; - - VALUE self = args[0]; - UNPACKER(self, mp); - - VALUE data = args[1]; - - size_t from = (unsigned long)args[2]; - char* dptr = RSTRING_PTR(data); - size_t dlen = (unsigned long)args[3]; - int ret; - - if(from >= dlen) { - rb_raise(eUnpackError, "offset is bigger than data buffer size."); - } - - mp->user.source = data; - ret = template_execute(mp, dptr, dlen, &from); - mp->user.source = Qnil; - - if(ret < 0) { - rb_raise(eUnpackError, "parse error."); - } else if(ret > 0) { - mp->user.finished = 1; - return ULONG2NUM(from); - } else { - mp->user.finished = 0; - return ULONG2NUM(from); - } -} - -static VALUE MessagePack_Unpacker_execute_rescue(VALUE nouse) -{ - rb_gc_enable(); -#ifdef RUBY_VM - rb_exc_raise(rb_errinfo()); -#else - rb_exc_raise(ruby_errinfo); -#endif -} - -static inline VALUE MessagePack_Unpacker_execute_impl(VALUE self, VALUE data, - unsigned long off, unsigned long dlen) -{ - // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ - rb_gc_disable(); - VALUE args[4] = {self, data, (VALUE)off, (VALUE)dlen}; - VALUE ret = rb_rescue(MessagePack_Unpacker_execute_do, (VALUE)args, - MessagePack_Unpacker_execute_rescue, Qnil); - rb_gc_enable(); - - return ret; -} - -static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, - VALUE off, VALUE limit) -{ - CHECK_STRING_TYPE(data); - return MessagePack_Unpacker_execute_impl(self, data, - NUM2ULONG(off), NUM2ULONG(limit)); -} - -static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) -{ - CHECK_STRING_TYPE(data); - return MessagePack_Unpacker_execute_impl(self, data, - NUM2ULONG(off), RSTRING_LEN(data)); -} - -static VALUE MessagePack_Unpacker_finished_p(VALUE self) -{ - UNPACKER(self, mp); - if(mp->user.finished) { - return Qtrue; - } - return Qfalse; -} - -static VALUE MessagePack_Unpacker_data(VALUE self) -{ - UNPACKER(self, mp); - return template_data(mp); -} - - -static VALUE MessagePack_Unpacker_feed(VALUE self, VALUE data) -{ - UNPACKER(self, mp); - StringValue(data); - rb_str_cat(mp->user.buffer, RSTRING_PTR(data), RSTRING_LEN(data)); - return Qnil; -} - static VALUE MessagePack_Unpacker_stream_get(VALUE self) { UNPACKER(self, mp); @@ -325,6 +282,14 @@ static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val) return val; } +static VALUE MessagePack_Unpacker_feed(VALUE self, VALUE data) +{ + UNPACKER(self, mp); + StringValue(data); + rb_str_cat(mp->user.buffer, RSTRING_PTR(data), RSTRING_LEN(data)); + return Qnil; +} + static VALUE MessagePack_Unpacker_fill(VALUE self) { UNPACKER(self, mp); @@ -333,7 +298,7 @@ static VALUE MessagePack_Unpacker_fill(VALUE self) return Qnil; } - size_t len; + long len; if(RSTRING_LEN(mp->user.buffer) == 0) { rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, LONG2FIX(64*1024), mp->user.buffer); @@ -368,16 +333,17 @@ static VALUE MessagePack_Unpacker_each(VALUE self) } } - mp->user.source = mp->user.buffer; - ret = template_execute(mp, RSTRING_PTR(mp->user.buffer), RSTRING_LEN(mp->user.buffer), &mp->user.offset); - mp->user.source = Qnil; + ret = template_execute_wrap(mp, mp->user.buffer, + RSTRING_LEN(mp->user.buffer), &mp->user.offset); if(ret < 0) { rb_raise(eUnpackError, "parse error."); + } else if(ret > 0) { VALUE data = template_data(mp); template_init(mp); rb_yield(data); + } else { goto do_fill; } @@ -386,70 +352,28 @@ static VALUE MessagePack_Unpacker_each(VALUE self) return Qnil; } - -static VALUE MessagePack_unpack_do(VALUE argv) -{ - VALUE* args = (VALUE*)argv; - - msgpack_unpack_t* mp = (msgpack_unpack_t*)args[0]; - VALUE data = args[1]; - - size_t from = 0; - char* dptr = RSTRING_PTR(data); - size_t dlen = (unsigned long)args[2]; - int ret; - - mp->user.source = data; - ret = template_execute(mp, dptr, dlen, &from); - mp->user.source = Qnil; - - if(ret < 0) { - rb_raise(eUnpackError, "parse error."); - } else if(ret == 0) { - rb_raise(eUnpackError, "insufficient bytes."); - } else { - if(from < dlen) { - rb_raise(eUnpackError, "extra bytes."); - } - return template_data(mp); - } -} - -static VALUE MessagePack_unpack_rescue(VALUE nouse) -{ - rb_gc_enable(); -#ifdef RUBY_VM - rb_exc_raise(rb_errinfo()); -#else - rb_exc_raise(ruby_errinfo); -#endif -} - static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned long dlen) { msgpack_unpack_t mp; template_init(&mp); - unpack_user u = {0, Qnil, 0, 0, Qnil, Qnil, Qnil}; - mp.user = u; -#ifdef HAVE_RUBY_ENCODING_H - // FIXME encodingã‚’ASCII-8BITã«ã™ã‚‹ - int enc_orig = rb_enc_get_index(data); - rb_enc_set_index(data, s_ascii_8bit); -#endif + mp.user.finished = 0; - // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ - rb_gc_disable(); - VALUE args[3] = {(VALUE)&mp, data, (VALUE)dlen}; - VALUE ret = rb_rescue(MessagePack_unpack_do, (VALUE)args, - MessagePack_unpack_rescue, Qnil); - rb_gc_enable(); + size_t from = 0; + int ret = template_execute_wrap(&mp, data, dlen, &from); -#ifdef HAVE_RUBY_ENCODING_H - rb_enc_set_index(data, enc_orig); -#endif + if(ret < 0) { + rb_raise(eUnpackError, "parse error."); - return ret; + } else if(ret == 0) { + rb_raise(eUnpackError, "insufficient bytes."); + + } else { + if(from < dlen) { + rb_raise(eUnpackError, "extra bytes."); + } + return template_data(&mp); + } } static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) @@ -465,6 +389,54 @@ static VALUE MessagePack_unpack(VALUE self, VALUE data) } +/* compat */ +static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, + VALUE off, VALUE limit) +{ + CHECK_STRING_TYPE(data); + UNPACKER(self, mp); + size_t from = (size_t)NUM2ULONG(off); + int ret = template_execute_wrap(mp, data, NUM2ULONG(limit), &from); + return INT2FIX(ret); +} + +/* compat */ +static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) +{ + CHECK_STRING_TYPE(data); + UNPACKER(self, mp); + size_t from = (size_t)NUM2ULONG(off); + int ret = template_execute_wrap(mp, data, RSTRING_LEN(data), &from); + return INT2FIX(ret); +} + +/* compat */ +static VALUE MessagePack_Unpacker_finished_p(VALUE self) +{ + UNPACKER(self, mp); + if(mp->user.finished) { + return Qtrue; + } + return Qfalse; +} + +/* compat */ +static VALUE MessagePack_Unpacker_data(VALUE self) +{ + UNPACKER(self, mp); + return template_data(mp); +} + +/* compat */ +static VALUE MessagePack_Unpacker_reset(VALUE self) +{ + UNPACKER(self, mp); + template_init(mp); + mp->user.finished = 0; + return self; +} + + void Init_msgpack_unpack(VALUE mMessagePack) { s_sysread = rb_intern("sysread"); @@ -478,11 +450,6 @@ void Init_msgpack_unpack(VALUE mMessagePack) cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); rb_define_method(cUnpacker, "initialize", MessagePack_Unpacker_initialize, -1); - rb_define_method(cUnpacker, "execute", MessagePack_Unpacker_execute, 2); - rb_define_method(cUnpacker, "execute_limit", MessagePack_Unpacker_execute_limit, 3); - rb_define_method(cUnpacker, "finished?", MessagePack_Unpacker_finished_p, 0); - rb_define_method(cUnpacker, "data", MessagePack_Unpacker_data, 0); - rb_define_method(cUnpacker, "reset", MessagePack_Unpacker_reset, 0); rb_define_method(cUnpacker, "feed", MessagePack_Unpacker_feed, 1); rb_define_method(cUnpacker, "fill", MessagePack_Unpacker_fill, 0); rb_define_method(cUnpacker, "each", MessagePack_Unpacker_each, 0); @@ -490,5 +457,12 @@ void Init_msgpack_unpack(VALUE mMessagePack) rb_define_method(cUnpacker, "stream=", MessagePack_Unpacker_stream_set, 1); rb_define_module_function(mMessagePack, "unpack", MessagePack_unpack, 1); rb_define_module_function(mMessagePack, "unpack_limit", MessagePack_unpack_limit, 2); + + /* backward compatibility */ + rb_define_method(cUnpacker, "execute", MessagePack_Unpacker_execute, 2); + rb_define_method(cUnpacker, "execute_limit", MessagePack_Unpacker_execute_limit, 3); + rb_define_method(cUnpacker, "finished?", MessagePack_Unpacker_finished_p, 0); + rb_define_method(cUnpacker, "data", MessagePack_Unpacker_data, 0); + rb_define_method(cUnpacker, "reset", MessagePack_Unpacker_reset, 0); } From d24193630ee5f3b5fcf5fd00bcaf95106b7a9c6d Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 22 Apr 2010 14:46:54 +0900 Subject: [PATCH 0420/1172] reverts variable-length stack: avoids memory leak --- cpp/unpack.c | 16 ++++------------ msgpack/unpack_define.h | 2 +- msgpack/unpack_template.h | 12 ++++++++++++ perl/unpack.c | 3 --- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/cpp/unpack.c b/cpp/unpack.c index 98c8653..34016fd 100644 --- a/cpp/unpack.c +++ b/cpp/unpack.c @@ -44,7 +44,6 @@ struct template_context; typedef struct template_context template_context; static void template_init(template_context* ctx); -static void template_destroy(template_context* ctx); static msgpack_object template_data(template_context* ctx); @@ -216,7 +215,6 @@ bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) void msgpack_unpacker_destroy(msgpack_unpacker* mpac) { msgpack_zone_free(mpac->z); - template_destroy(CTX_CAST(mpac->ctx)); free(mpac->ctx); decl_count(mpac->buffer); } @@ -370,7 +368,6 @@ msgpack_unpack_return msgpack_unpack(const char* data, size_t len, size_t* off, msgpack_zone* z, msgpack_object* result) { - msgpack_unpack_return ret = MSGPACK_UNPACK_SUCCESS; template_context ctx; template_init(&ctx); @@ -382,26 +379,21 @@ msgpack_unpack(const char* data, size_t len, size_t* off, int e = template_execute(&ctx, data, len, &noff); if(e < 0) { - ret = MSGPACK_UNPACK_PARSE_ERROR; - goto out; + return MSGPACK_UNPACK_PARSE_ERROR; } if(off != NULL) { *off = noff; } if(e == 0) { - ret = MSGPACK_UNPACK_CONTINUE; - goto out; + return MSGPACK_UNPACK_CONTINUE; } *result = template_data(&ctx); if(noff < len) { - ret = MSGPACK_UNPACK_EXTRA_BYTES; - goto out; + return MSGPACK_UNPACK_EXTRA_BYTES; } -out: - template_destroy(&ctx); - return ret; + return MSGPACK_UNPACK_SUCCESS; } diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 71412ee..959d351 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -30,7 +30,7 @@ extern "C" { #ifndef MSGPACK_EMBED_STACK_SIZE -#define MSGPACK_EMBED_STACK_SIZE 16 +#define MSGPACK_EMBED_STACK_SIZE 32 #endif diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 4b8cd14..0fbfbb7 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -58,9 +58,12 @@ msgpack_unpack_struct_decl(_context) { unsigned int cs; unsigned int trail; unsigned int top; + /* msgpack_unpack_struct(_stack)* stack; unsigned int stack_size; msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE]; + */ + msgpack_unpack_struct(_stack) stack[MSGPACK_EMBED_STACK_SIZE]; }; @@ -69,17 +72,21 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; + /* ctx->stack = ctx->embed_stack; ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; + */ ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); } +/* msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx) { if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { free(ctx->stack); } } +*/ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) { @@ -99,7 +106,9 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c unsigned int cs = ctx->cs; unsigned int top = ctx->top; msgpack_unpack_struct(_stack)* stack = ctx->stack; + /* unsigned int stack_size = ctx->stack_size; + */ msgpack_unpack_user* user = &ctx->user; msgpack_unpack_object obj; @@ -129,6 +138,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c goto _fixed_trail_again #define start_container(func, count_, ct_) \ + if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \ if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ if((count_) == 0) { obj = stack[top].obj; goto _push; } \ stack[top].ct = ct_; \ @@ -136,6 +146,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ++top; \ /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ /*printf("stack push %d\n", top);*/ \ + /* FIXME \ if(top >= stack_size) { \ if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \ size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \ @@ -153,6 +164,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ctx->stack_size = stack_size = stack_size * 2; \ } \ } \ + */ \ goto _header_again #define NEXT_CS(p) \ diff --git a/perl/unpack.c b/perl/unpack.c index c520e02..69017f1 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -53,7 +53,6 @@ struct template_context; typedef struct template_context msgpack_unpack_t; static void template_init(msgpack_unpack_t* u); -static void template_destroy(msgpack_unpack_t* u); static SV* template_data(msgpack_unpack_t* u); @@ -143,7 +142,6 @@ SV* _msgpack_unpack(SV* data, int limit) { mp.user.source = &PL_sv_undef; obj = template_data(&mp); - template_destroy(&mp); if(ret < 0) { Perl_croak(aTHX_ "parse error."); @@ -316,7 +314,6 @@ XS(xs_unpacker_destroy) { } UNPACKER(ST(0), mp); - template_destroy(mp); Safefree(mp); XSRETURN(0); From 60fbaf7612a5340de16ba98277ef01ffb80d1da0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 22 Apr 2010 14:56:25 +0900 Subject: [PATCH 0421/1172] ruby: 0.3.8 --- ruby/AUTHORS | 1 + ruby/ChangeLog | 0 ruby/makegem.sh | 4 +--- ruby/msgpack.gemspec | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 ruby/AUTHORS create mode 100644 ruby/ChangeLog diff --git a/ruby/AUTHORS b/ruby/AUTHORS new file mode 100644 index 0000000..ababacb --- /dev/null +++ b/ruby/AUTHORS @@ -0,0 +1 @@ +FURUHASHI Sadayuki diff --git a/ruby/ChangeLog b/ruby/ChangeLog new file mode 100644 index 0000000..e69de29 diff --git a/ruby/makegem.sh b/ruby/makegem.sh index 8737d29..5ea66f1 100755 --- a/ruby/makegem.sh +++ b/ruby/makegem.sh @@ -8,8 +8,6 @@ cp pack.h ext/ cp rbinit.c ext/ cp unpack.c ext/ cp unpack.h ext/ -cp ../AUTHORS ./ -cp ../ChangeLog ./ cp ../msgpack/pack_define.h msgpack/ cp ../msgpack/pack_template.h msgpack/ cp ../msgpack/unpack_define.h msgpack/ @@ -20,7 +18,7 @@ cat msgpack_test.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FI gem build msgpack.gemspec if [ $? -eq 0 ]; then - rm -rf ext msgpack AUTHORS ChangeLog test/msgpack_test.rb + rm -rf ext msgpack test/msgpack_test.rb fi # gem install gem-compile # on msys diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index 3a261f2..b8e88d5 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.3.7" + s.version = "0.3.8" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" From b10a7367440a33c9675be8fa435c9b202ff0c4f8 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 23 Apr 2010 18:13:36 +0900 Subject: [PATCH 0422/1172] ruby: fixese backward compatibility of streaming deserializer --- ruby/msgpack_test.rb | 22 ++++++++++++++++++++++ ruby/unpack.c | 34 ++++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/ruby/msgpack_test.rb b/ruby/msgpack_test.rb index 455aa40..45adf0b 100644 --- a/ruby/msgpack_test.rb +++ b/ruby/msgpack_test.rb @@ -218,6 +218,28 @@ class MessagePackTestFormat < Test::Unit::TestCase assert_equal(parsed, 1) end + it "streaming backward compatibility" do + obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"] + pac = MessagePack::Unpacker.new + buffer = "" + nread = 0 + parsed = 0 + obj.to_msgpack.split(//).each do |b| + buffer << b + nread = pac.execute(buffer, nread) + if pac.finished? + o = pac.data + assert_equal(obj, o) + parsed += 1 + pac.reset + buffer.slice!(0, nread) + nread = 0 + next unless buffer.empty? + end + end + assert_equal(parsed, 1) + end + private def check(len, obj) v = obj.to_msgpack diff --git a/ruby/unpack.c b/ruby/unpack.c index 73aeeeb..3a95e5a 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -389,25 +389,43 @@ static VALUE MessagePack_unpack(VALUE self, VALUE data) } +static VALUE MessagePack_Unpacker_execute_impl(VALUE self, VALUE data, + size_t from, size_t limit) +{ + UNPACKER(self, mp); + + if(from >= limit) { + rb_raise(eUnpackError, "offset is bigger than data buffer size."); + } + + int ret = template_execute_wrap(mp, data, limit, &from); + + if(ret < 0) { + rb_raise(eUnpackError, "parse error."); + } else if(ret > 0) { + mp->user.finished = 1; + return ULONG2NUM(from); + } else { + mp->user.finished = 0; + return ULONG2NUM(from); + } +} + /* compat */ static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, VALUE off, VALUE limit) { CHECK_STRING_TYPE(data); - UNPACKER(self, mp); - size_t from = (size_t)NUM2ULONG(off); - int ret = template_execute_wrap(mp, data, NUM2ULONG(limit), &from); - return INT2FIX(ret); + return MessagePack_Unpacker_execute_impl(self, data, + (size_t)NUM2ULONG(off), (size_t)NUM2ULONG(limit)); } /* compat */ static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) { CHECK_STRING_TYPE(data); - UNPACKER(self, mp); - size_t from = (size_t)NUM2ULONG(off); - int ret = template_execute_wrap(mp, data, RSTRING_LEN(data), &from); - return INT2FIX(ret); + return MessagePack_Unpacker_execute_impl(self, data, + (size_t)NUM2ULONG(off), (size_t)RSTRING_LEN(data)); } /* compat */ From c9fcf4020f309722e9f42a65bf7fe3b9c66a3a59 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 23 Apr 2010 20:18:48 +0900 Subject: [PATCH 0423/1172] ruby: streaming deserializer test --- ruby/msgpack_test.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ruby/msgpack_test.rb b/ruby/msgpack_test.rb index 45adf0b..8cbb586 100644 --- a/ruby/msgpack_test.rb +++ b/ruby/msgpack_test.rb @@ -204,9 +204,11 @@ class MessagePackTestFormat < Test::Unit::TestCase it "gc mark" do obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"] + num = 4 + raw = obj.to_msgpack * num pac = MessagePack::Unpacker.new parsed = 0 - obj.to_msgpack.split(//).each do |b| + raw.split(//).each do |b| pac.feed(b) pac.each {|o| GC.start @@ -215,16 +217,18 @@ class MessagePackTestFormat < Test::Unit::TestCase } GC.start end - assert_equal(parsed, 1) + assert_equal(parsed, num) end it "streaming backward compatibility" do obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"] + num = 4 + raw = obj.to_msgpack * num pac = MessagePack::Unpacker.new buffer = "" nread = 0 parsed = 0 - obj.to_msgpack.split(//).each do |b| + raw.split(//).each do |b| buffer << b nread = pac.execute(buffer, nread) if pac.finished? @@ -237,7 +241,7 @@ class MessagePackTestFormat < Test::Unit::TestCase next unless buffer.empty? end end - assert_equal(parsed, 1) + assert_equal(parsed, num) end private From 8335823748973b07dd628d3a94d49bdb615600ee Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 23 Apr 2010 20:24:36 +0900 Subject: [PATCH 0424/1172] ruby-0.3.9 --- ruby/msgpack.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index b8e88d5..c5e8c8c 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.3.8" + s.version = "0.3.9" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" From 120e8bffd7917e9529229e796b21ececc51df016 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 00:03:09 +0900 Subject: [PATCH 0425/1172] cpp: object::object(const T& v) and object::operator=(const T& v) --- cpp/msgpack/object.hpp | 107 +++++++++++++++---------------------- cpp/msgpack/type/bool.hpp | 6 +++ cpp/msgpack/type/float.hpp | 13 +++++ cpp/msgpack/type/int.hpp | 34 +++++++++++- cpp/msgpack/type/nil.hpp | 5 ++ 5 files changed, 101 insertions(+), 64 deletions(-) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index 3b42a8e..0125d0a 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -87,15 +87,15 @@ struct object { template void convert(T* v) const; - operator msgpack_object(); - object(msgpack_object obj); - object(); - object(bool v); - object(uint64_t v); - object(int64_t v); - object(double v); - object(const char* ptr, size_t size); + + template + object(const T& v); + + template + object& operator=(const T& v); + + operator msgpack_object(); private: struct implicit_type; @@ -115,9 +115,11 @@ bool operator!=(const object x, const object y); std::ostream& operator<< (std::ostream& s, const object o); +// serialize operator template packer& operator<< (packer& o, const T& v); +// convert operator template T& operator>> (object o, T& v); @@ -190,61 +192,6 @@ inline bool operator!=(const object x, const object y) { return !(x == y); } -inline object::object() -{ - type = type::NIL; -} - -inline object::object(bool v) -{ - type = type::BOOLEAN; - via.boolean = v; -} - -inline object::object(uint64_t v) -{ - type = type::POSITIVE_INTEGER; - via.u64 = v; -} - -inline object::object(int64_t v) -{ - if(v >= 0) { - type = type::POSITIVE_INTEGER; - via.u64 = v; - } else { - type = type::NEGATIVE_INTEGER; - via.i64 = v; - } -} - -inline object::object(double v) -{ - type = type::DOUBLE; - via.dec = v; -} - -inline object::object(const char* ptr, size_t size) -{ - type = type::RAW; - via.raw.size = size; - via.raw.ptr = ptr; -} - -inline object::object(msgpack_object obj) -{ - // FIXME beter way? - ::memcpy(this, &obj, sizeof(obj)); -} - -inline object::operator msgpack_object() -{ - // FIXME beter way? - msgpack_object obj; - ::memcpy(&obj, this, sizeof(obj)); - return obj; -} - inline object::implicit_type object::convert() const { return implicit_type(*this); @@ -265,6 +212,40 @@ inline T object::as() const } +inline object::object() +{ + type = type::NIL; +} + +template +inline object::object(const T& v) +{ + *this << v; +} + +template +inline object& object::operator=(const T& v) +{ + *this << v; + return *this; +} + + +inline object::operator msgpack_object() +{ + // FIXME beter way? + msgpack_object obj; + ::memcpy(&obj, this, sizeof(obj)); + return obj; +} + +inline void operator<< (object& o, msgpack_object v) +{ + // FIXME beter way? + ::memcpy(&o, &v, sizeof(v)); +} + + // obsolete template inline void convert(T& v, object o) diff --git a/cpp/msgpack/type/bool.hpp b/cpp/msgpack/type/bool.hpp index b945d85..9d63435 100644 --- a/cpp/msgpack/type/bool.hpp +++ b/cpp/msgpack/type/bool.hpp @@ -39,6 +39,12 @@ inline packer& operator<< (packer& o, const bool& v) return o; } +inline void operator<< (object& o, bool v) +{ + o.type = type::BOOLEAN; + o.via.boolean = v; +} + } // namespace msgpack diff --git a/cpp/msgpack/type/float.hpp b/cpp/msgpack/type/float.hpp index 390e340..28131d1 100644 --- a/cpp/msgpack/type/float.hpp +++ b/cpp/msgpack/type/float.hpp @@ -57,6 +57,19 @@ inline packer& operator<< (packer& o, const double& v) } +inline void operator<< (object& o, float v) +{ + o.type = type::DOUBLE; + o.via.dec = v; +} + +inline void operator<< (object& o, double v) +{ + o.type = type::DOUBLE; + o.via.dec = v; +} + + } // namespace msgpack #endif /* msgpack/type/float.hpp */ diff --git a/cpp/msgpack/type/int.hpp b/cpp/msgpack/type/int.hpp index 03500f7..2dab78b 100644 --- a/cpp/msgpack/type/int.hpp +++ b/cpp/msgpack/type/int.hpp @@ -56,7 +56,7 @@ namespace detail { throw type_error(); } }; - + template static inline T convert_integer(object o) { @@ -141,6 +141,38 @@ inline packer& operator<< (packer& o, const unsigned long long& { o.pack_unsigned_long_long(v); return o; } +inline void operator<< (object& o, signed char v) + { v < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v : o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, signed short v) + { v < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v : o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, signed int v) + { v < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v : o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, signed long v) + { v < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v : o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, signed long long v) + { v < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v : o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + + +inline void operator<< (object& o, unsigned char v) + { o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, unsigned short v) + { o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, unsigned int v) + { o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, unsigned long v) + { o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, unsigned long long v) + { o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + + } // namespace msgpack #endif /* msgpack/type/int.hpp */ diff --git a/cpp/msgpack/type/nil.hpp b/cpp/msgpack/type/nil.hpp index 93e66ff..ed58384 100644 --- a/cpp/msgpack/type/nil.hpp +++ b/cpp/msgpack/type/nil.hpp @@ -42,6 +42,11 @@ inline packer& operator<< (packer& o, const type::nil& v) return o; } +inline void operator<< (object& o, type::nil v) +{ + o.type = type::NIL; +} + } // namespace msgpack From 4e85ebbf9863201107a8f5c68fdabc92749e04fc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 01:12:25 +0900 Subject: [PATCH 0426/1172] cpp: object::object(const T& v, zone* z) --- cpp/msgpack/object.hpp | 25 +++++++++++++++++++++++++ cpp/msgpack/type/deque.hpp | 16 ++++++++++++++++ cpp/msgpack/type/list.hpp | 16 ++++++++++++++++ cpp/msgpack/type/pair.hpp | 11 +++++++++++ cpp/msgpack/type/raw.hpp | 7 +++++++ cpp/msgpack/type/set.hpp | 32 ++++++++++++++++++++++++++++++++ cpp/msgpack/type/string.hpp | 9 +++++++++ cpp/msgpack/type/tuple.hpp.erb | 20 ++++++++++++++++++++ cpp/msgpack/type/vector.hpp | 16 ++++++++++++++++ cpp/msgpack/zone.hpp.erb | 1 - 10 files changed, 152 insertions(+), 1 deletion(-) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index 0125d0a..a094c56 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -20,6 +20,7 @@ #include "msgpack/object.h" #include "msgpack/pack.hpp" +#include "msgpack/zone.hpp" #include #include #include @@ -92,6 +93,9 @@ struct object { template object(const T& v); + template + object(const T& v, zone* z); + template object& operator=(const T& v); @@ -100,6 +104,10 @@ struct object { private: struct implicit_type; +public: + // FIXME private? + struct object_zone; + public: implicit_type convert() const; }; @@ -109,6 +117,14 @@ struct object_kv { object val; }; +struct object::object_zone : object { + object_zone(msgpack::zone* zone) : zone(zone) { } + msgpack::zone* zone; +private: + object_zone(); +}; + + bool operator==(const object x, const object y); bool operator!=(const object x, const object y); @@ -230,6 +246,15 @@ inline object& object::operator=(const T& v) return *this; } +template +object::object(const T& v, zone* z) +{ + object_zone oz(z); + oz << v; + type = oz.type; + via = oz.via; +} + inline object::operator msgpack_object() { diff --git a/cpp/msgpack/type/deque.hpp b/cpp/msgpack/type/deque.hpp index d34d243..5e919a8 100644 --- a/cpp/msgpack/type/deque.hpp +++ b/cpp/msgpack/type/deque.hpp @@ -49,6 +49,22 @@ inline packer& operator<< (packer& o, const std::deque& v) return o; } +template +inline void operator<< (object::object_zone& o, const std::deque& v) +{ + o.type = type::ARRAY; + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::deque::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); +} + } // namespace msgpack diff --git a/cpp/msgpack/type/list.hpp b/cpp/msgpack/type/list.hpp index 6ecc02f..5318639 100644 --- a/cpp/msgpack/type/list.hpp +++ b/cpp/msgpack/type/list.hpp @@ -49,6 +49,22 @@ inline packer& operator<< (packer& o, const std::list& v) return o; } +template +inline void operator<< (object::object_zone& o, const std::list& v) +{ + o.type = type::ARRAY; + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::list::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); +} + } // namespace msgpack diff --git a/cpp/msgpack/type/pair.hpp b/cpp/msgpack/type/pair.hpp index ba72c1f..6c68288 100644 --- a/cpp/msgpack/type/pair.hpp +++ b/cpp/msgpack/type/pair.hpp @@ -43,6 +43,17 @@ inline packer& operator<< (packer& o, const std::pair& v return o; } +template +inline void operator<< (object::object_zone& o, const std::pair& v) +{ + o.type = type::ARRAY; + object* p = (object*)o.zone->malloc(sizeof(object)*2); + o.via.array.ptr = p; + o.via.array.size = 2; + p[0] = object(v.first, o.zone); + p[1] = object(v.second, o.zone); +} + } // namespace msgpack diff --git a/cpp/msgpack/type/raw.hpp b/cpp/msgpack/type/raw.hpp index 8c68aba..d854d28 100644 --- a/cpp/msgpack/type/raw.hpp +++ b/cpp/msgpack/type/raw.hpp @@ -77,6 +77,13 @@ inline packer& operator<< (packer& o, const type::raw_ref& v) return o; } +inline void operator<< (object& o, const type::raw_ref& v) +{ + o.type = type::RAW; + o.via.raw.ptr = v.ptr; + o.via.raw.size = v.size; +} + } // namespace msgpack diff --git a/cpp/msgpack/type/set.hpp b/cpp/msgpack/type/set.hpp index f2c5bfb..6f5fb18 100644 --- a/cpp/msgpack/type/set.hpp +++ b/cpp/msgpack/type/set.hpp @@ -48,6 +48,22 @@ inline packer& operator<< (packer& o, const std::set& v) return o; } +template +inline void operator<< (object::object_zone& o, const std::set& v) +{ + o.type = type::ARRAY; + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::set::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); +} + template inline std::multiset& operator>> (object o, std::multiset& v) @@ -73,6 +89,22 @@ inline packer& operator<< (packer& o, const std::multiset& v) return o; } +template +inline void operator<< (object::object_zone& o, const std::multiset& v) +{ + o.type = type::ARRAY; + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::multiset::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); +} + } // namespace msgpack diff --git a/cpp/msgpack/type/string.hpp b/cpp/msgpack/type/string.hpp index a085d53..796603f 100644 --- a/cpp/msgpack/type/string.hpp +++ b/cpp/msgpack/type/string.hpp @@ -39,6 +39,15 @@ inline packer& operator<< (packer& o, const std::string& v) return o; } +inline void operator<< (object::object_zone& o, const std::string& v) +{ + o.type = type::RAW; + char* ptr = (char*)o.zone->malloc(v.size()); + o.via.raw.ptr = ptr; + o.via.raw.size = v.size(); + memcpy(ptr, v.data(), v.size()); +} + } // namespace msgpack diff --git a/cpp/msgpack/type/tuple.hpp.erb b/cpp/msgpack/type/tuple.hpp.erb index 2930ae0..e57fe98 100644 --- a/cpp/msgpack/type/tuple.hpp.erb +++ b/cpp/msgpack/type/tuple.hpp.erb @@ -165,6 +165,26 @@ const packer& operator<< ( } <%}%> +inline void operator<< ( + object::object_zone& o, + const type::tuple<>& v) { + o.type = type::ARRAY; + o.via.array.ptr = NULL; + o.via.array.size = 0; +} +<%0.upto(GENERATION_LIMIT) {|i|%> +template , typename A<%=j%><%}%>> +inline void operator<< ( + object::object_zone& o, + const type::tuple, A<%=j%><%}%>>& v) { + o.type = type::ARRAY; + o.via.array.ptr = (object*)o.zone->malloc(sizeof(object)*<%=i+1%>); + o.via.array.size = <%=i+1%>; + <%0.upto(i) {|j|%> + o.via.array.ptr[<%=j%>] = object(v.template get<<%=j%>>(), o.zone);<%}%> +} +<%}%> + } // namespace msgpack #endif /* msgpack/type/tuple.hpp */ diff --git a/cpp/msgpack/type/vector.hpp b/cpp/msgpack/type/vector.hpp index 385d070..f9e709b 100644 --- a/cpp/msgpack/type/vector.hpp +++ b/cpp/msgpack/type/vector.hpp @@ -53,6 +53,22 @@ inline packer& operator<< (packer& o, const std::vector& v) return o; } +template +inline void operator<< (object::object_zone& o, const std::vector& v) +{ + o.type = type::ARRAY; + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::vector::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); +} + } // namespace msgpack diff --git a/cpp/msgpack/zone.hpp.erb b/cpp/msgpack/zone.hpp.erb index 48988ab..8e69aa4 100644 --- a/cpp/msgpack/zone.hpp.erb +++ b/cpp/msgpack/zone.hpp.erb @@ -18,7 +18,6 @@ #ifndef MSGPACK_ZONE_HPP__ #define MSGPACK_ZONE_HPP__ -#include "msgpack/object.hpp" #include "msgpack/zone.h" #include #include From 01b6673528127538457fdf445531f3574dc0c20a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 01:24:24 +0900 Subject: [PATCH 0427/1172] cpp: bool operator==(object& x, const T& y) --- cpp/msgpack/object.hpp | 11 +++++++++++ cpp/msgpack/type/string.hpp | 7 +++++++ cpp/object.cpp | 3 +++ 3 files changed, 21 insertions(+) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index a094c56..df1079b 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -128,6 +128,9 @@ private: bool operator==(const object x, const object y); bool operator!=(const object x, const object y); +template +bool operator==(const object x, const T& y); + std::ostream& operator<< (std::ostream& s, const object o); @@ -207,6 +210,14 @@ inline packer& operator<< (packer& o, const T& v) inline bool operator!=(const object x, const object y) { return !(x == y); } +template +inline bool operator==(const object x, const T& y) +try { + return x == object(y); +} catch (msgpack::type_error&) { + return false; +} + inline object::implicit_type object::convert() const { diff --git a/cpp/msgpack/type/string.hpp b/cpp/msgpack/type/string.hpp index 796603f..8ca2724 100644 --- a/cpp/msgpack/type/string.hpp +++ b/cpp/msgpack/type/string.hpp @@ -48,6 +48,13 @@ inline void operator<< (object::object_zone& o, const std::string& v) memcpy(ptr, v.data(), v.size()); } +inline void operator<< (object& o, const std::string& v) +{ + o.type = type::RAW; + o.via.raw.ptr = v.data(); + o.via.raw.size = v.size(); +} + } // namespace msgpack diff --git a/cpp/object.cpp b/cpp/object.cpp index a7adb71..626d186 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -103,6 +103,9 @@ bool operator==(const object x, const object y) case type::NEGATIVE_INTEGER: return x.via.i64 == y.via.i64; + case type::DOUBLE: + return x.via.dec == y.via.dec; + case type::RAW: return x.via.raw.size == y.via.raw.size && memcmp(x.via.raw.ptr, y.via.raw.ptr, x.via.raw.size) == 0; From 05e28752f111f6faa8b61432a7063a30b9f51ebf Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 01:57:05 +0900 Subject: [PATCH 0428/1172] cpp: MSGPACK_DEFINE defines T::msgpack_object(object*, zone*) --- cpp/msgpack/object.hpp | 22 ++++++++++++++++------ cpp/msgpack/type/bool.hpp | 3 +++ cpp/msgpack/type/define.hpp.erb | 18 ++++++++++++++++++ cpp/msgpack/type/deque.hpp | 2 +- cpp/msgpack/type/float.hpp | 6 ++++++ cpp/msgpack/type/int.hpp | 32 ++++++++++++++++++++++++++++++++ cpp/msgpack/type/list.hpp | 2 +- cpp/msgpack/type/nil.hpp | 3 +++ cpp/msgpack/type/pair.hpp | 2 +- cpp/msgpack/type/raw.hpp | 3 +++ cpp/msgpack/type/set.hpp | 4 ++-- cpp/msgpack/type/string.hpp | 2 +- cpp/msgpack/type/tuple.hpp.erb | 4 ++-- cpp/msgpack/type/vector.hpp | 2 +- 14 files changed, 90 insertions(+), 15 deletions(-) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index df1079b..bfbaeb1 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -106,7 +106,7 @@ private: public: // FIXME private? - struct object_zone; + struct with_zone; public: implicit_type convert() const; @@ -117,11 +117,11 @@ struct object_kv { object val; }; -struct object::object_zone : object { - object_zone(msgpack::zone* zone) : zone(zone) { } +struct object::with_zone : object { + with_zone(msgpack::zone* zone) : zone(zone) { } msgpack::zone* zone; private: - object_zone(); + with_zone(); }; @@ -142,6 +142,10 @@ packer& operator<< (packer& o, const T& v); template T& operator>> (object o, T& v); +// deconvert operator +template +void operator<< (object::with_zone& o, const T& v); + struct object::implicit_type { implicit_type(object o) : obj(o) { } @@ -206,6 +210,12 @@ inline packer& operator<< (packer& o, const T& v) return o; } +template +void operator<< (object::with_zone& o, const T& v) +{ + v.msgpack_object(static_cast(&o), o.zone); +} + inline bool operator!=(const object x, const object y) { return !(x == y); } @@ -253,14 +263,14 @@ inline object::object(const T& v) template inline object& object::operator=(const T& v) { - *this << v; + *this = object(v); return *this; } template object::object(const T& v, zone* z) { - object_zone oz(z); + with_zone oz(z); oz << v; type = oz.type; via = oz.via; diff --git a/cpp/msgpack/type/bool.hpp b/cpp/msgpack/type/bool.hpp index 9d63435..9433a98 100644 --- a/cpp/msgpack/type/bool.hpp +++ b/cpp/msgpack/type/bool.hpp @@ -45,6 +45,9 @@ inline void operator<< (object& o, bool v) o.via.boolean = v; } +inline void operator<< (object::with_zone& o, bool v) + { static_cast(o) << v; } + } // namespace msgpack diff --git a/cpp/msgpack/type/define.hpp.erb b/cpp/msgpack/type/define.hpp.erb index 2eac4f4..369e12f 100644 --- a/cpp/msgpack/type/define.hpp.erb +++ b/cpp/msgpack/type/define.hpp.erb @@ -27,6 +27,10 @@ void msgpack_unpack(msgpack::object o) \ { \ msgpack::type::make_define(__VA_ARGS__).msgpack_unpack(o); \ + }\ + void msgpack_object(msgpack::object* o, msgpack::zone* z) const \ + { \ + msgpack::type::make_define(__VA_ARGS__).msgpack_object(o, z); \ } namespace msgpack { @@ -51,6 +55,12 @@ struct define<> { { if(o.type != type::ARRAY) { throw type_error(); } } + void msgpack_object(msgpack::object* o, msgpack::zone* z) const + { + o->type = type::ARRAY; + o->via.array.ptr = NULL; + o->via.array.size = 0; + } }; <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> @@ -73,6 +83,14 @@ struct define, A<%=j%><%}%>> { <%0.upto(i) {|j|%> if(size <= <%=j%>) { return; } o.via.array.ptr[<%=j%>].convert(&a<%=j%>);<%}%> } + void msgpack_object(msgpack::object* o, msgpack::zone* z) const + { + o->type = type::ARRAY; + o->via.array.ptr = (object*)z->malloc(sizeof(object)*<%=i+1%>); + o->via.array.size = <%=i+1%>; + <%0.upto(i) {|j|%> + o->via.array.ptr[<%=j%>] = object(a<%=j%>, z);<%}%> + } <%0.upto(i) {|j|%> A<%=j%>& a<%=j%>;<%}%> }; diff --git a/cpp/msgpack/type/deque.hpp b/cpp/msgpack/type/deque.hpp index 5e919a8..6ae00a8 100644 --- a/cpp/msgpack/type/deque.hpp +++ b/cpp/msgpack/type/deque.hpp @@ -50,7 +50,7 @@ inline packer& operator<< (packer& o, const std::deque& v) } template -inline void operator<< (object::object_zone& o, const std::deque& v) +inline void operator<< (object::with_zone& o, const std::deque& v) { o.type = type::ARRAY; object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); diff --git a/cpp/msgpack/type/float.hpp b/cpp/msgpack/type/float.hpp index 28131d1..a60ef0b 100644 --- a/cpp/msgpack/type/float.hpp +++ b/cpp/msgpack/type/float.hpp @@ -69,6 +69,12 @@ inline void operator<< (object& o, double v) o.via.dec = v; } +inline void operator<< (object::with_zone& o, float v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, double v) + { static_cast(o) << v; } + } // namespace msgpack diff --git a/cpp/msgpack/type/int.hpp b/cpp/msgpack/type/int.hpp index 2dab78b..e2d1820 100644 --- a/cpp/msgpack/type/int.hpp +++ b/cpp/msgpack/type/int.hpp @@ -173,6 +173,38 @@ inline void operator<< (object& o, unsigned long long v) { o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } +inline void operator<< (object::with_zone& o, signed char v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, signed short v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, signed int v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, signed long v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, signed long long v) + { static_cast(o) << v; } + + +inline void operator<< (object::with_zone& o, unsigned char v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, unsigned short v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, unsigned int v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, unsigned long v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, unsigned long long v) + { static_cast(o) << v; } + + } // namespace msgpack #endif /* msgpack/type/int.hpp */ diff --git a/cpp/msgpack/type/list.hpp b/cpp/msgpack/type/list.hpp index 5318639..6807345 100644 --- a/cpp/msgpack/type/list.hpp +++ b/cpp/msgpack/type/list.hpp @@ -50,7 +50,7 @@ inline packer& operator<< (packer& o, const std::list& v) } template -inline void operator<< (object::object_zone& o, const std::list& v) +inline void operator<< (object::with_zone& o, const std::list& v) { o.type = type::ARRAY; object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); diff --git a/cpp/msgpack/type/nil.hpp b/cpp/msgpack/type/nil.hpp index ed58384..e58bc9d 100644 --- a/cpp/msgpack/type/nil.hpp +++ b/cpp/msgpack/type/nil.hpp @@ -47,6 +47,9 @@ inline void operator<< (object& o, type::nil v) o.type = type::NIL; } +inline void operator<< (object::with_zone& o, type::nil v) + { static_cast(o) << v; } + } // namespace msgpack diff --git a/cpp/msgpack/type/pair.hpp b/cpp/msgpack/type/pair.hpp index 6c68288..296a8b6 100644 --- a/cpp/msgpack/type/pair.hpp +++ b/cpp/msgpack/type/pair.hpp @@ -44,7 +44,7 @@ inline packer& operator<< (packer& o, const std::pair& v } template -inline void operator<< (object::object_zone& o, const std::pair& v) +inline void operator<< (object::with_zone& o, const std::pair& v) { o.type = type::ARRAY; object* p = (object*)o.zone->malloc(sizeof(object)*2); diff --git a/cpp/msgpack/type/raw.hpp b/cpp/msgpack/type/raw.hpp index d854d28..21d9a0d 100644 --- a/cpp/msgpack/type/raw.hpp +++ b/cpp/msgpack/type/raw.hpp @@ -84,6 +84,9 @@ inline void operator<< (object& o, const type::raw_ref& v) o.via.raw.size = v.size; } +inline void operator<< (object::with_zone& o, const type::raw_ref& v) + { static_cast(o) << v; } + } // namespace msgpack diff --git a/cpp/msgpack/type/set.hpp b/cpp/msgpack/type/set.hpp index 6f5fb18..2a6e7a4 100644 --- a/cpp/msgpack/type/set.hpp +++ b/cpp/msgpack/type/set.hpp @@ -49,7 +49,7 @@ inline packer& operator<< (packer& o, const std::set& v) } template -inline void operator<< (object::object_zone& o, const std::set& v) +inline void operator<< (object::with_zone& o, const std::set& v) { o.type = type::ARRAY; object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); @@ -90,7 +90,7 @@ inline packer& operator<< (packer& o, const std::multiset& v) } template -inline void operator<< (object::object_zone& o, const std::multiset& v) +inline void operator<< (object::with_zone& o, const std::multiset& v) { o.type = type::ARRAY; object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); diff --git a/cpp/msgpack/type/string.hpp b/cpp/msgpack/type/string.hpp index 8ca2724..f11a5e6 100644 --- a/cpp/msgpack/type/string.hpp +++ b/cpp/msgpack/type/string.hpp @@ -39,7 +39,7 @@ inline packer& operator<< (packer& o, const std::string& v) return o; } -inline void operator<< (object::object_zone& o, const std::string& v) +inline void operator<< (object::with_zone& o, const std::string& v) { o.type = type::RAW; char* ptr = (char*)o.zone->malloc(v.size()); diff --git a/cpp/msgpack/type/tuple.hpp.erb b/cpp/msgpack/type/tuple.hpp.erb index e57fe98..1b0c172 100644 --- a/cpp/msgpack/type/tuple.hpp.erb +++ b/cpp/msgpack/type/tuple.hpp.erb @@ -166,7 +166,7 @@ const packer& operator<< ( <%}%> inline void operator<< ( - object::object_zone& o, + object::with_zone& o, const type::tuple<>& v) { o.type = type::ARRAY; o.via.array.ptr = NULL; @@ -175,7 +175,7 @@ inline void operator<< ( <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> inline void operator<< ( - object::object_zone& o, + object::with_zone& o, const type::tuple, A<%=j%><%}%>>& v) { o.type = type::ARRAY; o.via.array.ptr = (object*)o.zone->malloc(sizeof(object)*<%=i+1%>); diff --git a/cpp/msgpack/type/vector.hpp b/cpp/msgpack/type/vector.hpp index f9e709b..382f501 100644 --- a/cpp/msgpack/type/vector.hpp +++ b/cpp/msgpack/type/vector.hpp @@ -54,7 +54,7 @@ inline packer& operator<< (packer& o, const std::vector& v) } template -inline void operator<< (object::object_zone& o, const std::vector& v) +inline void operator<< (object::with_zone& o, const std::vector& v) { o.type = type::ARRAY; object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); From 7d945d3c8e24de21ba5862d699521502bce17fe5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 02:30:53 +0900 Subject: [PATCH 0429/1172] cpp: explicit object(const T& v) --- cpp/msgpack/object.hpp | 37 +++++++++++++++++++++++++++++---- cpp/msgpack/type/define.hpp.erb | 3 ++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index bfbaeb1..3567ac8 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -90,8 +90,10 @@ struct object { object(); + object(msgpack_object o); + template - object(const T& v); + explicit object(const T& v); template object(const T& v, zone* z); @@ -131,6 +133,15 @@ bool operator!=(const object x, const object y); template bool operator==(const object x, const T& y); +template +bool operator==(const T& y, const object x); + +template +bool operator!=(const object x, const T& y); + +template +bool operator!=(const T& y, const object x); + std::ostream& operator<< (std::ostream& s, const object o); @@ -217,9 +228,6 @@ void operator<< (object::with_zone& o, const T& v) } -inline bool operator!=(const object x, const object y) -{ return !(x == y); } - template inline bool operator==(const object x, const T& y) try { @@ -228,6 +236,21 @@ try { return false; } +inline bool operator!=(const object x, const object y) +{ return !(x == y); } + +template +inline bool operator==(const T& y, const object x) +{ return x == y; } + +template +inline bool operator!=(const object x, const T& y) +{ return !(x == y); } + +template +inline bool operator!=(const T& y, const object x) +{ return x != y; } + inline object::implicit_type object::convert() const { @@ -277,6 +300,12 @@ object::object(const T& v, zone* z) } +inline object::object(msgpack_object o) +{ + // FIXME beter way? + ::memcpy(this, &o, sizeof(o)); +} + inline object::operator msgpack_object() { // FIXME beter way? diff --git a/cpp/msgpack/type/define.hpp.erb b/cpp/msgpack/type/define.hpp.erb index 369e12f..9db6f08 100644 --- a/cpp/msgpack/type/define.hpp.erb +++ b/cpp/msgpack/type/define.hpp.erb @@ -28,7 +28,8 @@ { \ msgpack::type::make_define(__VA_ARGS__).msgpack_unpack(o); \ }\ - void msgpack_object(msgpack::object* o, msgpack::zone* z) const \ + template \ + void msgpack_object(MSGPACK_OBJECT* o, msgpack::zone* z) const \ { \ msgpack::type::make_define(__VA_ARGS__).msgpack_object(o, z); \ } From 9bfa2354ff9e667436fc4e6ced2274a5c0e55aab Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 02:37:04 +0900 Subject: [PATCH 0430/1172] cpp: fixes serialization of object::type == DOUBLE --- cpp/msgpack/object.hpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index 3567ac8..37c1926 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -306,6 +306,12 @@ inline object::object(msgpack_object o) ::memcpy(this, &o, sizeof(o)); } +inline void operator<< (object& o, msgpack_object v) +{ + // FIXME beter way? + ::memcpy(&o, &v, sizeof(v)); +} + inline object::operator msgpack_object() { // FIXME beter way? @@ -314,12 +320,6 @@ inline object::operator msgpack_object() return obj; } -inline void operator<< (object& o, msgpack_object v) -{ - // FIXME beter way? - ::memcpy(&o, &v, sizeof(v)); -} - // obsolete template @@ -391,6 +391,10 @@ packer& operator<< (packer& o, const object& v) } return o; + case type::DOUBLE: + o.pack_double(v.via.dec); + return o; + case type::RAW: o.pack_raw(v.via.raw.size); o.pack_raw_body(v.via.raw.ptr, v.via.raw.size); From 9df6916029e2eba4ca0f93578f631d34208e6415 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 06:39:12 +0900 Subject: [PATCH 0431/1172] cpp: object::object(const T& v, zone* z) 2 --- cpp/msgpack/object.hpp | 6 +-- cpp/msgpack/type/deque.hpp | 25 ++++++---- cpp/msgpack/type/list.hpp | 25 ++++++---- cpp/msgpack/type/map.hpp | 66 ++++++++++++++++++++++++++ cpp/msgpack/type/set.hpp | 50 +++++++++++-------- cpp/msgpack/type/tr1/unordered_map.hpp | 44 +++++++++++++++++ cpp/msgpack/type/tr1/unordered_set.hpp | 42 ++++++++++++++++ cpp/msgpack/type/vector.hpp | 25 ++++++---- 8 files changed, 229 insertions(+), 54 deletions(-) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index 37c1926..a72913a 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -103,13 +103,11 @@ struct object { operator msgpack_object(); + struct with_zone; + private: struct implicit_type; -public: - // FIXME private? - struct with_zone; - public: implicit_type convert() const; }; diff --git a/cpp/msgpack/type/deque.hpp b/cpp/msgpack/type/deque.hpp index 6ae00a8..d21ceea 100644 --- a/cpp/msgpack/type/deque.hpp +++ b/cpp/msgpack/type/deque.hpp @@ -53,16 +53,21 @@ template inline void operator<< (object::with_zone& o, const std::deque& v) { o.type = type::ARRAY; - object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); - object* const pend = p + v.size(); - o.via.array.ptr = p; - o.via.array.size = v.size(); - typename std::deque::const_iterator it(v.begin()); - do { - *p = object(*it, o.zone); - ++p; - ++it; - } while(p < pend); + if(v.empty()) { + o.via.array.ptr = NULL; + o.via.array.size = 0; + } else { + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::deque::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); + } } diff --git a/cpp/msgpack/type/list.hpp b/cpp/msgpack/type/list.hpp index 6807345..c0f8ce6 100644 --- a/cpp/msgpack/type/list.hpp +++ b/cpp/msgpack/type/list.hpp @@ -53,16 +53,21 @@ template inline void operator<< (object::with_zone& o, const std::list& v) { o.type = type::ARRAY; - object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); - object* const pend = p + v.size(); - o.via.array.ptr = p; - o.via.array.size = v.size(); - typename std::list::const_iterator it(v.begin()); - do { - *p = object(*it, o.zone); - ++p; - ++it; - } while(p < pend); + if(v.empty()) { + o.via.array.ptr = NULL; + o.via.array.size = 0; + } else { + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::list::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); + } } diff --git a/cpp/msgpack/type/map.hpp b/cpp/msgpack/type/map.hpp index 552de57..958447d 100644 --- a/cpp/msgpack/type/map.hpp +++ b/cpp/msgpack/type/map.hpp @@ -70,6 +70,28 @@ inline packer& operator<< (packer& o, const type::assoc_vector +inline void operator<< (object::with_zone& o, const type::assoc_vector& v) +{ + o.type = type::MAP; + if(v.empty()) { + o.via.map.ptr = NULL; + o.via.map.size = 0; + } else { + object_kv* p = (object_kv*)o.zone->malloc(sizeof(object_kv)*v.size()); + object_kv* const pend = p + v.size(); + o.via.map.ptr = p; + o.via.map.size = v.size(); + typename type::assoc_vector::const_iterator it(v.begin()); + do { + p->key = object(it->first, o.zone); + p->val = object(it->second, o.zone); + ++p; + ++it; + } while(p < pend); + } +} + template inline std::map operator>> (object o, std::map& v) @@ -104,6 +126,28 @@ inline packer& operator<< (packer& o, const std::map& v) return o; } +template +inline void operator<< (object::with_zone& o, const std::map& v) +{ + o.type = type::MAP; + if(v.empty()) { + o.via.map.ptr = NULL; + o.via.map.size = 0; + } else { + object_kv* p = (object_kv*)o.zone->malloc(sizeof(object_kv)*v.size()); + object_kv* const pend = p + v.size(); + o.via.map.ptr = p; + o.via.map.size = v.size(); + typename std::map::const_iterator it(v.begin()); + do { + p->key = object(it->first, o.zone); + p->val = object(it->second, o.zone); + ++p; + ++it; + } while(p < pend); + } +} + template inline std::multimap operator>> (object o, std::multimap& v) @@ -132,6 +176,28 @@ inline packer& operator<< (packer& o, const std::multimap& return o; } +template +inline void operator<< (object::with_zone& o, const std::multimap& v) +{ + o.type = type::MAP; + if(v.empty()) { + o.via.map.ptr = NULL; + o.via.map.size = 0; + } else { + object_kv* p = (object_kv*)o.zone->malloc(sizeof(object_kv)*v.size()); + object_kv* const pend = p + v.size(); + o.via.map.ptr = p; + o.via.map.size = v.size(); + typename std::multimap::const_iterator it(v.begin()); + do { + p->key = object(it->first, o.zone); + p->val = object(it->second, o.zone); + ++p; + ++it; + } while(p < pend); + } +} + } // namespace msgpack diff --git a/cpp/msgpack/type/set.hpp b/cpp/msgpack/type/set.hpp index 2a6e7a4..bcf1030 100644 --- a/cpp/msgpack/type/set.hpp +++ b/cpp/msgpack/type/set.hpp @@ -52,16 +52,21 @@ template inline void operator<< (object::with_zone& o, const std::set& v) { o.type = type::ARRAY; - object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); - object* const pend = p + v.size(); - o.via.array.ptr = p; - o.via.array.size = v.size(); - typename std::set::const_iterator it(v.begin()); - do { - *p = object(*it, o.zone); - ++p; - ++it; - } while(p < pend); + if(v.empty()) { + o.via.array.ptr = NULL; + o.via.array.size = 0; + } else { + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::set::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); + } } @@ -93,16 +98,21 @@ template inline void operator<< (object::with_zone& o, const std::multiset& v) { o.type = type::ARRAY; - object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); - object* const pend = p + v.size(); - o.via.array.ptr = p; - o.via.array.size = v.size(); - typename std::multiset::const_iterator it(v.begin()); - do { - *p = object(*it, o.zone); - ++p; - ++it; - } while(p < pend); + if(v.empty()) { + o.via.array.ptr = NULL; + o.via.array.size = 0; + } else { + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::multiset::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); + } } diff --git a/cpp/msgpack/type/tr1/unordered_map.hpp b/cpp/msgpack/type/tr1/unordered_map.hpp index 1996cfd..4b29f0c 100644 --- a/cpp/msgpack/type/tr1/unordered_map.hpp +++ b/cpp/msgpack/type/tr1/unordered_map.hpp @@ -50,6 +50,28 @@ inline packer& operator<< (packer& o, const std::tr1::unordered_ return o; } +template +inline void operator<< (object::with_zone& o, const std::tr1::unordered_map& v) +{ + o.type = type::MAP; + if(v.empty()) { + o.via.map.ptr = NULL; + o.via.map.size = 0; + } else { + object_kv* p = (object_kv*)o.zone->malloc(sizeof(object_kv)*v.size()); + object_kv* const pend = p + v.size(); + o.via.map.ptr = p; + o.via.map.size = v.size(); + typename std::tr1::unordered_map::const_iterator it(v.begin()); + do { + p->key = object(it->first, o.zone); + p->val = object(it->second, o.zone); + ++p; + ++it; + } while(p < pend); + } +} + template inline std::tr1::unordered_multimap operator>> (object o, std::tr1::unordered_multimap& v) @@ -78,6 +100,28 @@ inline packer& operator<< (packer& o, const std::tr1::unordered_ return o; } +template +inline void operator<< (object::with_zone& o, const std::tr1::unordered_multimap& v) +{ + o.type = type::MAP; + if(v.empty()) { + o.via.map.ptr = NULL; + o.via.map.size = 0; + } else { + object_kv* p = (object_kv*)o.zone->malloc(sizeof(object_kv)*v.size()); + object_kv* const pend = p + v.size(); + o.via.map.ptr = p; + o.via.map.size = v.size(); + typename std::tr1::unordered_multimap::const_iterator it(v.begin()); + do { + p->key = object(it->first, o.zone); + p->val = object(it->second, o.zone); + ++p; + ++it; + } while(p < pend); + } +} + } // namespace msgpack diff --git a/cpp/msgpack/type/tr1/unordered_set.hpp b/cpp/msgpack/type/tr1/unordered_set.hpp index eb127b5..4af6801 100644 --- a/cpp/msgpack/type/tr1/unordered_set.hpp +++ b/cpp/msgpack/type/tr1/unordered_set.hpp @@ -48,6 +48,27 @@ inline packer& operator<< (packer& o, const std::tr1::unordered_ return o; } +template +inline void operator<< (object::with_zone& o, const std::tr1::unordered_set& v) +{ + o.type = type::ARRAY; + if(v.empty()) { + o.via.array.ptr = NULL; + o.via.array.size = 0; + } else { + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::tr1::unordered_set::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); + } +} + template inline std::tr1::unordered_multiset& operator>> (object o, std::tr1::unordered_multiset& v) @@ -73,6 +94,27 @@ inline packer& operator<< (packer& o, const std::tr1::unordered_ return o; } +template +inline void operator<< (object::with_zone& o, const std::tr1::unordered_multiset& v) +{ + o.type = type::ARRAY; + if(v.empty()) { + o.via.array.ptr = NULL; + o.via.array.size = 0; + } else { + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::tr1::unordered_multiset::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); + } +} + } // namespace msgpack diff --git a/cpp/msgpack/type/vector.hpp b/cpp/msgpack/type/vector.hpp index 382f501..bd073ef 100644 --- a/cpp/msgpack/type/vector.hpp +++ b/cpp/msgpack/type/vector.hpp @@ -57,16 +57,21 @@ template inline void operator<< (object::with_zone& o, const std::vector& v) { o.type = type::ARRAY; - object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); - object* const pend = p + v.size(); - o.via.array.ptr = p; - o.via.array.size = v.size(); - typename std::vector::const_iterator it(v.begin()); - do { - *p = object(*it, o.zone); - ++p; - ++it; - } while(p < pend); + if(v.empty()) { + o.via.array.ptr = NULL; + o.via.array.size = 0; + } else { + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::vector::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); + } } From 72160aac9a1536234700539eacc9bdc7bf6f9f61 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 07:56:19 +0900 Subject: [PATCH 0432/1172] cpp: combines libmsgpackc and libmsgpack into libmsgpack --- cpp/Makefile.am | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index a297ba7..f1a1d6c 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -1,22 +1,15 @@ -lib_LTLIBRARIES = libmsgpackc.la libmsgpack.la - -libmsgpackc_la_SOURCES = \ - unpack.c \ - object.c \ - vrefbuffer.c \ - zone.c - -# -version-info CURRENT:REVISION:AGE -libmsgpackc_la_LDFLAGS = -version-info 2:0:0 +lib_LTLIBRARIES = libmsgpack.la libmsgpack_la_SOURCES = \ + unpack.c \ + objectc.c \ + vrefbuffer.c \ + zone.c \ object.cpp -libmsgpack_la_LIBADD = -lmsgpackc - # -version-info CURRENT:REVISION:AGE -libmsgpack_la_LDFLAGS = -version-info 2:0:0 +libmsgpack_la_LDFLAGS = -version-info 3:0:0 nobase_include_HEADERS = \ @@ -60,13 +53,6 @@ nobase_include_HEADERS = \ msgpack/type/tr1/unordered_set.hpp -# work around for duplicated object file name -libmsgpackc_la_CFLAGS = $(AM_CFLAGS) -libmsgpackc_la_CXXFLAGS = $(AM_CXXFLAGS) -libmsgpack_la_CFLAGS = $(AM_CFLAGS) -libmsgpack_la_CXXFLAGS = $(AM_CXXFLAGS) - - EXTRA_DIST = \ README.md \ LICENSE \ From 35802ba94948463db19308c8ec385243c51d1485 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 08:00:04 +0900 Subject: [PATCH 0433/1172] cpp: msgpack_object_equal --- cpp/Makefile.am | 2 +- cpp/msgpack/object.h | 2 ++ cpp/msgpack/object.hpp | 9 +++-- cpp/object.cpp | 54 ---------------------------- cpp/{object.c => objectc.c} | 70 +++++++++++++++++++++++++++++++++++-- 5 files changed, 78 insertions(+), 59 deletions(-) rename cpp/{object.c => objectc.c} (71%) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index f1a1d6c..fa8d6cf 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -66,7 +66,7 @@ check_PROGRAMS = \ msgpack_test msgpackc_test_SOURCES = msgpackc_test.cpp -msgpackc_test_LDADD = libmsgpackc.la -lgtest_main +msgpackc_test_LDADD = libmsgpack.la -lgtest_main msgpack_test_SOURCES = msgpack_test.cpp msgpack_test_LDADD = libmsgpack.la -lgtest_main diff --git a/cpp/msgpack/object.h b/cpp/msgpack/object.h index 9a014be..bbfac19 100644 --- a/cpp/msgpack/object.h +++ b/cpp/msgpack/object.h @@ -79,6 +79,8 @@ typedef struct msgpack_object_kv { void msgpack_object_print(FILE* out, msgpack_object o); +bool msgpack_object_equal(const msgpack_object x, const msgpack_object y); + #ifdef __cplusplus } diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index a72913a..1362818 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -101,7 +101,7 @@ struct object { template object& operator=(const T& v); - operator msgpack_object(); + operator msgpack_object() const; struct with_zone; @@ -226,6 +226,11 @@ void operator<< (object::with_zone& o, const T& v) } +inline bool operator==(const object x, const object y) +{ + return msgpack_object_equal(x, y); +} + template inline bool operator==(const object x, const T& y) try { @@ -310,7 +315,7 @@ inline void operator<< (object& o, msgpack_object v) ::memcpy(&o, &v, sizeof(v)); } -inline object::operator msgpack_object() +inline object::operator msgpack_object() const { // FIXME beter way? msgpack_object obj; diff --git a/cpp/object.cpp b/cpp/object.cpp index 626d186..dfe32bb 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -16,7 +16,6 @@ // limitations under the License. // #include "msgpack/object.hpp" -#include namespace msgpack { @@ -61,7 +60,6 @@ std::ostream& operator<< (std::ostream& s, const object o) } s << "]"; break; - // FIXME loop optimiziation case type::MAP: s << "{"; @@ -76,7 +74,6 @@ std::ostream& operator<< (std::ostream& s, const object o) } s << "}"; break; - // FIXME loop optimiziation default: // FIXME @@ -86,56 +83,5 @@ std::ostream& operator<< (std::ostream& s, const object o) } -bool operator==(const object x, const object y) -{ - if(x.type != y.type) { return false; } - - switch(x.type) { - case type::NIL: - return true; - - case type::BOOLEAN: - return x.via.boolean == y.via.boolean; - - case type::POSITIVE_INTEGER: - return x.via.u64 == y.via.u64; - - case type::NEGATIVE_INTEGER: - return x.via.i64 == y.via.i64; - - case type::DOUBLE: - return x.via.dec == y.via.dec; - - case type::RAW: - return x.via.raw.size == y.via.raw.size && - memcmp(x.via.raw.ptr, y.via.raw.ptr, x.via.raw.size) == 0; - - case type::ARRAY: - if(x.via.array.size != y.via.array.size) { return false; } - for(object* px(x.via.array.ptr), - * const pxend(x.via.array.ptr + x.via.array.size), - * py(y.via.array.ptr); - px < pxend; ++px, ++py) { - if(*px != *py) { return false; } - } - return true; - // FIXME loop optimiziation - - case type::MAP: - if(x.via.map.size != y.via.map.size) { return false; } - for(object_kv* px(x.via.map.ptr), - * const pxend(x.via.map.ptr + x.via.map.size), - * py(y.via.map.ptr); - px < pxend; ++px, ++py) { - if(px->key != py->key || px->val != py->val) { return false; } - } - return true; - - default: - return false; - } -} - - } // namespace msgpack diff --git a/cpp/object.c b/cpp/objectc.c similarity index 71% rename from cpp/object.c rename to cpp/objectc.c index a22ce21..d4f1c8a 100644 --- a/cpp/object.c +++ b/cpp/objectc.c @@ -18,6 +18,7 @@ #include "msgpack/object.h" #include "msgpack/pack.h" #include +#include #ifndef _MSC_VER #include @@ -141,7 +142,6 @@ void msgpack_object_print(FILE* out, msgpack_object o) } fprintf(out, "]"); break; - // FIXME loop optimiziation case MSGPACK_OBJECT_MAP: fprintf(out, "{"); @@ -161,7 +161,6 @@ void msgpack_object_print(FILE* out, msgpack_object o) } fprintf(out, "}"); break; - // FIXME loop optimiziation default: // FIXME @@ -169,3 +168,70 @@ void msgpack_object_print(FILE* out, msgpack_object o) } } +bool msgpack_object_equal(const msgpack_object x, const msgpack_object y) +{ + if(x.type != y.type) { return false; } + + switch(x.type) { + case MSGPACK_OBJECT_NIL: + return true; + + case MSGPACK_OBJECT_BOOLEAN: + return x.via.boolean == y.via.boolean; + + case MSGPACK_OBJECT_POSITIVE_INTEGER: + return x.via.u64 == y.via.u64; + + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + return x.via.i64 == y.via.i64; + + case MSGPACK_OBJECT_DOUBLE: + return x.via.dec == y.via.dec; + + case MSGPACK_OBJECT_RAW: + return x.via.raw.size == y.via.raw.size && + memcmp(x.via.raw.ptr, y.via.raw.ptr, x.via.raw.size) == 0; + + case MSGPACK_OBJECT_ARRAY: + if(x.via.array.size != y.via.array.size) { + return false; + } else if(x.via.array.size == 0) { + return true; + } else { + msgpack_object* px = x.via.array.ptr; + msgpack_object* const pxend = x.via.array.ptr + x.via.array.size; + msgpack_object* py = y.via.array.ptr; + do { + if(!msgpack_object_equal(*px, *py)) { + return false; + } + ++px; + ++py; + } while(px < pxend); + return true; + } + + case MSGPACK_OBJECT_MAP: + if(x.via.map.size != y.via.map.size) { + return false; + } else if(x.via.map.size == 0) { + return true; + } else { + msgpack_object_kv* px = x.via.map.ptr; + msgpack_object_kv* const pxend = x.via.map.ptr + x.via.map.size; + msgpack_object_kv* py = y.via.map.ptr; + do { + if(!msgpack_object_equal(px->key, py->key) || !msgpack_object_equal(px->val, py->val)) { + return false; + } + ++px; + ++py; + } while(px < pxend); + return true; + } + + default: + return false; + } +} + From 0a5c2e7ab9ade5b9789782f10cc51eff9cacc0a9 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 08:16:12 +0900 Subject: [PATCH 0434/1172] c,cpp: MSGPACK_OBJECT_NIL = 0x00 --- cpp/msgpack/object.h | 16 ++++++++-------- cpp/msgpack/object.hpp | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cpp/msgpack/object.h b/cpp/msgpack/object.h index bbfac19..71a27bb 100644 --- a/cpp/msgpack/object.h +++ b/cpp/msgpack/object.h @@ -27,14 +27,14 @@ extern "C" { typedef enum { - MSGPACK_OBJECT_NIL = 0x01, - MSGPACK_OBJECT_BOOLEAN = 0x02, - MSGPACK_OBJECT_POSITIVE_INTEGER = 0x03, - MSGPACK_OBJECT_NEGATIVE_INTEGER = 0x04, - MSGPACK_OBJECT_DOUBLE = 0x05, - MSGPACK_OBJECT_RAW = 0x06, - MSGPACK_OBJECT_ARRAY = 0x07, - MSGPACK_OBJECT_MAP = 0x08, + MSGPACK_OBJECT_NIL = 0x00, + MSGPACK_OBJECT_BOOLEAN = 0x01, + MSGPACK_OBJECT_POSITIVE_INTEGER = 0x02, + MSGPACK_OBJECT_NEGATIVE_INTEGER = 0x03, + MSGPACK_OBJECT_DOUBLE = 0x04, + MSGPACK_OBJECT_RAW = 0x05, + MSGPACK_OBJECT_ARRAY = 0x06, + MSGPACK_OBJECT_MAP = 0x07, } msgpack_object_type; diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index 1362818..4df4437 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -35,14 +35,14 @@ class type_error : public std::bad_cast { }; namespace type { enum object_type { - NIL = 0x01, - BOOLEAN = 0x02, - POSITIVE_INTEGER = 0x03, - NEGATIVE_INTEGER = 0x04, - DOUBLE = 0x05, - RAW = 0x06, - ARRAY = 0x07, - MAP = 0x08, + NIL = MSGPACK_OBJECT_NIL, + BOOLEAN = MSGPACK_OBJECT_BOOLEAN, + POSITIVE_INTEGER = MSGPACK_OBJECT_POSITIVE_INTEGER, + NEGATIVE_INTEGER = MSGPACK_OBJECT_NEGATIVE_INTEGER, + DOUBLE = MSGPACK_OBJECT_DOUBLE, + RAW = MSGPACK_OBJECT_RAW, + ARRAY = MSGPACK_OBJECT_ARRAY, + MAP = MSGPACK_OBJECT_MAP, }; } From 53d5ddb3455dceb98c97b18026ad436d180a8b19 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 08:26:42 +0900 Subject: [PATCH 0435/1172] cpp: fixes operator<<(packer&, const object&) --- cpp/msgpack/object.hpp | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index 4df4437..f80a390 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -363,35 +363,11 @@ packer& operator<< (packer& o, const object& v) return o; case type::POSITIVE_INTEGER: - if(v.via.u64 <= (uint64_t)std::numeric_limits::max()) { - if(v.via.u64 <= (uint16_t)std::numeric_limits::max()) { - o.pack_uint8(v.via.u64); - } else { - o.pack_uint16(v.via.u64); - } - } else { - if(v.via.u64 <= (uint64_t)std::numeric_limits::max()) { - o.pack_uint32(v.via.u64); - } else { - o.pack_uint64(v.via.u64); - } - } + o.pack_uint64(v.via.u64); return o; case type::NEGATIVE_INTEGER: - if(v.via.i64 >= (int64_t)std::numeric_limits::min()) { - if(v.via.i64 >= (int64_t)std::numeric_limits::min()) { - o.pack_int8(v.via.i64); - } else { - o.pack_int16(v.via.i64); - } - } else { - if(v.via.i64 >= (int64_t)std::numeric_limits::min()) { - o.pack_int64(v.via.i64); - } else { - o.pack_int64(v.via.i64); - } - } + o.pack_int64(v.via.i64); return o; case type::DOUBLE: From d19bfaa2cb16eaabee79922f7d8b59ef9bccb592 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 09:09:06 +0900 Subject: [PATCH 0436/1172] cpp: fixes msgpack_vc8.vcproj --- cpp/msgpack_vc8.vcproj | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/cpp/msgpack_vc8.vcproj b/cpp/msgpack_vc8.vcproj index 2a0eebe..5804790 100644 --- a/cpp/msgpack_vc8.vcproj +++ b/cpp/msgpack_vc8.vcproj @@ -157,15 +157,13 @@ UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}" > @@ -174,16 +172,10 @@ > - - @@ -244,6 +236,10 @@ /> + + Date: Sun, 25 Apr 2010 18:08:14 +0900 Subject: [PATCH 0437/1172] cpp: add test/{zone,pack_unpack,streaming,object,convert,buffer}.cc --- cpp/Makefile.am | 2 + cpp/configure.in | 2 +- cpp/msgpack/unpack.hpp | 2 +- cpp/test/Makefile.am | 26 ++++++++ cpp/test/buffer.cc | 50 +++++++++++++++ cpp/test/convert.cc | 74 ++++++++++++++++++++++ cpp/test/object.cc | 134 ++++++++++++++++++++++++++++++++++++++++ cpp/test/pack_unpack.cc | 101 ++++++++++++++++++++++++++++++ cpp/test/streaming.cc | 109 ++++++++++++++++++++++++++++++++ cpp/test/zone.cc | 78 +++++++++++++++++++++++ 10 files changed, 576 insertions(+), 2 deletions(-) create mode 100644 cpp/test/Makefile.am create mode 100644 cpp/test/buffer.cc create mode 100644 cpp/test/convert.cc create mode 100644 cpp/test/object.cc create mode 100644 cpp/test/pack_unpack.cc create mode 100644 cpp/test/streaming.cc create mode 100644 cpp/test/zone.cc diff --git a/cpp/Makefile.am b/cpp/Makefile.am index fa8d6cf..3cbb400 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -61,6 +61,8 @@ EXTRA_DIST = \ msgpack_vc8.sln \ msgpack_vc8.postbuild.bat +SUBDIRS = test + check_PROGRAMS = \ msgpackc_test \ msgpack_test diff --git a/cpp/configure.in b/cpp/configure.in index 8c59dda..596b28d 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -39,5 +39,5 @@ add CFLAGS="--march=i686" and CXXFLAGS="-march=i686" options to ./configure as f ]) fi -AC_OUTPUT([Makefile]) +AC_OUTPUT([Makefile test/Makefile]) diff --git a/cpp/msgpack/unpack.hpp b/cpp/msgpack/unpack.hpp index 324111a..d39b5df 100644 --- a/cpp/msgpack/unpack.hpp +++ b/cpp/msgpack/unpack.hpp @@ -81,7 +81,7 @@ public: // while( /* readable */ ) { // // // 1. - // pac.reserve(1024); + // pac.reserve_buffer(1024); // // // 2. // ssize_t bytes = diff --git a/cpp/test/Makefile.am b/cpp/test/Makefile.am new file mode 100644 index 0000000..a80f319 --- /dev/null +++ b/cpp/test/Makefile.am @@ -0,0 +1,26 @@ + +AM_LDFLAGS = ../libmsgpack.la -lgtest_main + +check_PROGRAMS = \ + zone \ + pack_unpack \ + streaming \ + object \ + convert \ + buffer + +TESTS = $(check_PROGRAMS) + +zone_SOURCES = zone.cc + +pack_unpack_SOURCES = pack_unpack.cc + +streaming_SOURCES = streaming.cc + +object_SOURCES = object.cc + +convert_SOURCES = convert.cc + +buffer_SOURCES = buffer.cc +buffer_LDADD = -lz + diff --git a/cpp/test/buffer.cc b/cpp/test/buffer.cc new file mode 100644 index 0000000..a2e9037 --- /dev/null +++ b/cpp/test/buffer.cc @@ -0,0 +1,50 @@ +#include +#include +#include +#include + +TEST(buffer, sbuffer) +{ + msgpack::sbuffer sbuf; + sbuf.write("a", 1); + sbuf.write("a", 1); + sbuf.write("a", 1); + + EXPECT_EQ(3, sbuf.size()); + EXPECT_TRUE( memcmp(sbuf.data(), "aaa", 3) == 0 ); +} + + +TEST(buffer, vrefbuffer) +{ + msgpack::vrefbuffer vbuf; + vbuf.write("a", 1); + vbuf.write("a", 1); + vbuf.write("a", 1); + + const struct iovec* vec = vbuf.vector(); + size_t veclen = vbuf.vector_size(); + + msgpack::sbuffer sbuf; + for(size_t i=0; i < veclen; ++i) { + sbuf.write((const char*)vec[i].iov_base, vec[i].iov_len); + } + + EXPECT_EQ(3, sbuf.size()); + EXPECT_TRUE( memcmp(sbuf.data(), "aaa", 3) == 0 ); +} + + +TEST(buffer, zbuffer) +{ + msgpack::zbuffer zbuf; + zbuf.write("a", 1); + zbuf.write("a", 1); + zbuf.write("a", 1); + + zbuf.flush(); + + char* data = zbuf.data(); + size_t size = zbuf.size(); +} + diff --git a/cpp/test/convert.cc b/cpp/test/convert.cc new file mode 100644 index 0000000..f2a8523 --- /dev/null +++ b/cpp/test/convert.cc @@ -0,0 +1,74 @@ +#include +#include + +class compatibility { +public: + compatibility() : str1("default"), str2("default") { } + + std::string str1; + std::string str2; + + MSGPACK_DEFINE(str1, str2); +}; + +TEST(convert, compatibility_less) +{ + std::vector src(1); + src[0] = "kumofs"; + + msgpack::zone z; + msgpack::object obj(src, &z); + + compatibility c; + EXPECT_NO_THROW( obj.convert(&c) ); + + EXPECT_EQ("kumofs", c.str1); + EXPECT_EQ("default", c.str2); +} + +TEST(convert, compatibility_more) +{ + std::vector src(3); + src[0] = "kumofs"; + src[1] = "mpio"; + src[2] = "cloudy"; + + msgpack::zone z; + msgpack::object obj(src, &z); + + compatibility to; + EXPECT_NO_THROW( obj.convert(&to) ); + + EXPECT_EQ("kumofs", to.str1); + EXPECT_EQ("mpio", to.str2); +} + + +class enum_member { +public: + enum_member() : flag(A) { } + + enum flags_t { + A = 0, + B = 1, + }; + + flags_t flag; + + MSGPACK_DEFINE((int&)flag); +}; + +TEST(convert, enum_member) +{ + enum_member src; + src.flag = enum_member::B; + + msgpack::zone z; + msgpack::object obj(src, &z); + + enum_member to; + EXPECT_NO_THROW( obj.convert(&to) ); + + EXPECT_EQ(enum_member::B, to.flag); +} + diff --git a/cpp/test/object.cc b/cpp/test/object.cc new file mode 100644 index 0000000..5390c4a --- /dev/null +++ b/cpp/test/object.cc @@ -0,0 +1,134 @@ +#include +#include + +struct myclass { + myclass() : num(0), str("default") { } + + myclass(int num, const std::string& str) : + num(0), str("default") { } + + ~myclass() { } + + int num; + std::string str; + + MSGPACK_DEFINE(num, str); + + bool operator==(const myclass& o) const + { + return num == o.num && str == o.str; + } +}; + +std::ostream& operator<<(std::ostream& o, const myclass& m) +{ + return o << "myclass("<()); +} + + +TEST(object, print) +{ + msgpack::object obj; + std::cout << obj << std::endl; +} + + +TEST(object, is_nil) +{ + msgpack::object obj; + EXPECT_TRUE(obj.is_nil()); +} + + +TEST(object, type_error) +{ + msgpack::object obj(1); + EXPECT_THROW(obj.as(), msgpack::type_error); + EXPECT_THROW(obj.as >(), msgpack::type_error); + EXPECT_EQ(1, obj.as()); + EXPECT_EQ(1, obj.as()); + EXPECT_EQ(1u, obj.as()); + EXPECT_EQ(1u, obj.as()); +} + + +TEST(object, equal_primitive) +{ + msgpack::object obj_nil; + EXPECT_EQ(obj_nil, msgpack::object()); + + msgpack::object obj_int(1); + EXPECT_EQ(obj_int, msgpack::object(1)); + EXPECT_EQ(obj_int, 1); + + msgpack::object obj_double(1.2); + EXPECT_EQ(obj_double, msgpack::object(1.2)); + EXPECT_EQ(obj_double, 1.2); + + msgpack::object obj_bool(true); + EXPECT_EQ(obj_bool, msgpack::object(true)); + EXPECT_EQ(obj_bool, true); +} + + +TEST(object, construct_primitive) +{ + msgpack::object obj_nil; + EXPECT_EQ(msgpack::type::NIL, obj_nil.type); + + msgpack::object obj_uint(1); + EXPECT_EQ(msgpack::type::POSITIVE_INTEGER, obj_uint.type); + EXPECT_EQ(1u, obj_uint.via.u64); + + msgpack::object obj_int(-1); + EXPECT_EQ(msgpack::type::NEGATIVE_INTEGER, obj_int.type); + EXPECT_EQ(-1, obj_int.via.i64); + + msgpack::object obj_double(1.2); + EXPECT_EQ(msgpack::type::DOUBLE, obj_double.type); + EXPECT_EQ(1.2, obj_double.via.dec); + + msgpack::object obj_bool(true); + EXPECT_EQ(msgpack::type::BOOLEAN, obj_bool.type); + EXPECT_EQ(true, obj_bool.via.boolean); +} + diff --git a/cpp/test/pack_unpack.cc b/cpp/test/pack_unpack.cc new file mode 100644 index 0000000..ecf52c5 --- /dev/null +++ b/cpp/test/pack_unpack.cc @@ -0,0 +1,101 @@ +#include +#include +#include + +TEST(pack, num) +{ + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, 1); +} + + +TEST(pack, vector) +{ + msgpack::sbuffer sbuf; + std::vector vec; + vec.push_back(1); + vec.push_back(2); + vec.push_back(3); + msgpack::pack(sbuf, vec); +} + + +TEST(pack, to_ostream) +{ + std::ostringstream stream; + msgpack::pack(stream, 1); +} + + +struct myclass { + myclass() : num(0), str("default") { } + + myclass(int num, const std::string& str) : + num(0), str("default") { } + + ~myclass() { } + + int num; + std::string str; + + MSGPACK_DEFINE(num, str); +}; + + +TEST(pack, myclass) +{ + msgpack::sbuffer sbuf; + myclass m(1, "msgpack"); + msgpack::pack(sbuf, m); +} + + +TEST(unpack, myclass) +{ + msgpack::sbuffer sbuf; + myclass m1(1, "phraser"); + msgpack::pack(sbuf, m1); + + msgpack::zone z; + msgpack::object obj; + + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + + myclass m2 = obj.as(); + EXPECT_EQ(m1.num, m2.num); + EXPECT_EQ(m1.str, m2.str); +} + + +TEST(unpack, sequence) +{ + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, 1); + msgpack::pack(sbuf, 2); + msgpack::pack(sbuf, 3); + + size_t offset = 0; + + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret; + + ret = msgpack::unpack(sbuf.data(), sbuf.size(), &offset, &z, &obj); + EXPECT_TRUE(ret >= 0); + EXPECT_EQ(ret, msgpack::UNPACK_EXTRA_BYTES); + EXPECT_EQ(1, obj.as()); + + ret = msgpack::unpack(sbuf.data(), sbuf.size(), &offset, &z, &obj); + EXPECT_TRUE(ret >= 0); + EXPECT_EQ(ret, msgpack::UNPACK_EXTRA_BYTES); + EXPECT_EQ(2, obj.as()); + + ret = msgpack::unpack(sbuf.data(), sbuf.size(), &offset, &z, &obj); + EXPECT_TRUE(ret >= 0); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(3, obj.as()); +} + diff --git a/cpp/test/streaming.cc b/cpp/test/streaming.cc new file mode 100644 index 0000000..2d03976 --- /dev/null +++ b/cpp/test/streaming.cc @@ -0,0 +1,109 @@ +#include +#include +#include + +TEST(streaming, basic) +{ + std::ostringstream stream; + msgpack::packer pk(&stream); + + pk.pack(1); + pk.pack(2); + pk.pack(3); + + std::istringstream input(stream.str()); + + msgpack::unpacker pac; + + int count = 0; + while(count < 3) { + pac.reserve_buffer(32*1024); + + size_t len = input.readsome(pac.buffer(), pac.buffer_capacity()); + pac.buffer_consumed(len); + + while(pac.execute()) { + std::auto_ptr z(pac.release_zone()); + msgpack::object obj = pac.data(); + pac.reset(); + + switch(count++) { + case 0: + EXPECT_EQ(1, obj.as()); + break; + case 1: + EXPECT_EQ(2, obj.as()); + break; + case 2: + EXPECT_EQ(3, obj.as()); + return; + } + + } + } +} + + +class event_handler { +public: + event_handler(std::istream& input) : input(input) { } + ~event_handler() { } + + void on_read() + { + while(true) { + pac.reserve_buffer(32*1024); + + size_t len = input.readsome(pac.buffer(), pac.buffer_capacity()); + + if(len == 0) { + return; + } + + pac.buffer_consumed(len); + + while(pac.execute()) { + std::auto_ptr z(pac.release_zone()); + msgpack::object obj = pac.data(); + pac.reset(); + on_message(obj, z); + } + + if(pac.message_size() > 10*1024*1024) { + throw std::runtime_error("message is too large"); + } + } + } + + void on_message(msgpack::object obj, std::auto_ptr z) + { + EXPECT_EQ(expect, obj.as()); + } + + int expect; + +private: + std::istream& input; + msgpack::unpacker pac; +}; + +TEST(streaming, event) +{ + std::stringstream stream; + msgpack::packer pk(&stream); + + event_handler handler(stream); + + pk.pack(1); + handler.expect = 1; + handler.on_read(); + + pk.pack(2); + handler.expect = 2; + handler.on_read(); + + pk.pack(3); + handler.expect = 3; + handler.on_read(); +} + diff --git a/cpp/test/zone.cc b/cpp/test/zone.cc new file mode 100644 index 0000000..5274e9f --- /dev/null +++ b/cpp/test/zone.cc @@ -0,0 +1,78 @@ +#include +#include + +TEST(zone, malloc) +{ + msgpack::zone z; + char* buf1 = (char*)z.malloc(4); + memcpy(buf1, "test", 4); + char* buf2 = (char*)z.malloc(4); + memcpy(buf2, "test", 4); +} + + +class myclass { +public: + myclass() : num(0), str("default") { } + + myclass(int num, const std::string& str) : + num(num), str(str) { } + + ~myclass() { } + + int num; + std::string str; + +private: + myclass(const myclass&); +}; + + +TEST(zone, allocate) +{ + msgpack::zone z; + myclass* m = z.allocate(); + EXPECT_EQ(m->num, 0); + EXPECT_EQ(m->str, "default"); +} + + +TEST(zone, allocate_constructor) +{ + msgpack::zone z; + myclass* m = z.allocate(7, "msgpack"); + EXPECT_EQ(m->num, 7); + EXPECT_EQ(m->str, "msgpack"); +} + + +static void custom_finalizer_func(void* user) +{ + myclass* m = (myclass*)user; + delete m; +} + +TEST(zone, push_finalizer) +{ + msgpack::zone z; + myclass* m = new myclass(); + z.push_finalizer(custom_finalizer_func, (void*)m); +} + + +TEST(zone, push_finalizer_auto_ptr) +{ + msgpack::zone z; + std::auto_ptr am(new myclass()); + z.push_finalizer(am); +} + + +TEST(zone, malloc_no_align) +{ + msgpack::zone z; + char* buf1 = (char*)z.malloc_no_align(4); + char* buf2 = (char*)z.malloc_no_align(4); + EXPECT_EQ(buf1+4, buf2); +} + From 68f60568acdafb2205c1436b61bfcaf0087b1572 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 26 Apr 2010 21:52:19 +0900 Subject: [PATCH 0438/1172] cpp: build libmsgpackc.so for backward compatibility. --- cpp/Makefile.am | 16 ++++++++++++++++ cpp/test/Makefile.am | 14 ++++++++------ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 3cbb400..08eb7a5 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -12,6 +12,22 @@ libmsgpack_la_SOURCES = \ libmsgpack_la_LDFLAGS = -version-info 3:0:0 +# backward compatibility +lib_LTLIBRARIES += libmsgpackc.la + +libmsgpackc_la_SOURCES = \ + unpack.c \ + objectc.c \ + vrefbuffer.c \ + zone.c + +libmsgpackc_la_LDFLAGS = -version-info 2:0:0 + +# work around for duplicated file name +kumo_manager_CFLAGS = $(AM_CFLAGS) +kumo_manager_CXXFLAGS = $(AM_CXXFLAGS) + + nobase_include_HEADERS = \ msgpack/pack_define.h \ msgpack/pack_template.h \ diff --git a/cpp/test/Makefile.am b/cpp/test/Makefile.am index a80f319..2b96669 100644 --- a/cpp/test/Makefile.am +++ b/cpp/test/Makefile.am @@ -1,13 +1,15 @@ +AM_CPPFLAGS = -I.. +AM_C_CPPFLAGS = -I.. AM_LDFLAGS = ../libmsgpack.la -lgtest_main check_PROGRAMS = \ - zone \ - pack_unpack \ - streaming \ - object \ - convert \ - buffer + zone \ + pack_unpack \ + streaming \ + object \ + convert \ + buffer TESTS = $(check_PROGRAMS) From b10cb658caa245d9c21abbe94f32fde02e8cba43 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 29 Apr 2010 00:39:45 +0900 Subject: [PATCH 0439/1172] pack_template.h: template_unsigned_long: wrong size checking on !defined(SIZEOF_SHORT) && !defined(SHRT_MAX)" --- msgpack/pack_template.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index f8f522f..c0a1073 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -512,9 +512,9 @@ msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long msgpack_pack_real_uint64(x, d); #endif #else -if(sizeof(unsigned int) == 2) { +if(sizeof(unsigned long) == 2) { msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned int) == 4) { +} else if(sizeof(unsigned long) == 4) { msgpack_pack_real_uint32(x, d); } else { msgpack_pack_real_uint64(x, d); From c51fabf6ed70180dce2557dc2987c261e9449c03 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 29 Apr 2010 02:39:53 +0900 Subject: [PATCH 0440/1172] msgpack/pack_template.h: don't evaluate undefined macro --- msgpack/pack_template.h | 132 ++++++++++++++++++++++++++++++++-------- 1 file changed, 106 insertions(+), 26 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index c0a1073..daa8f81 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -16,12 +16,12 @@ * limitations under the License. */ -#ifdef __LITTLE_ENDIAN__ +#if defined(__LITTLE_ENDIAN__) #define TAKE8_8(d) ((uint8_t*)&d)[0] #define TAKE8_16(d) ((uint8_t*)&d)[0] #define TAKE8_32(d) ((uint8_t*)&d)[0] #define TAKE8_64(d) ((uint8_t*)&d)[0] -#elif __BIG_ENDIAN__ +#elif defined(__BIG_ENDIAN__) #define TAKE8_8(d) ((uint8_t*)&d)[0] #define TAKE8_16(d) ((uint8_t*)&d)[1] #define TAKE8_32(d) ((uint8_t*)&d)[3] @@ -377,14 +377,24 @@ msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d) msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d) { -#if defined(SIZEOF_SHORT) || defined(SHRT_MAX) -#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff +#if defined(SIZEOF_SHORT) +#if SIZEOF_SHORT == 2 msgpack_pack_real_int16(x, d); -#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff +#elif SIZEOF_SHORT == 4 msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif + +#elif defined(SHRT_MAX) +#if SHRT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SHRT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif + #else if(sizeof(short) == 2) { msgpack_pack_real_int16(x, d); @@ -398,14 +408,24 @@ if(sizeof(short) == 2) { msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d) { -#if defined(SIZEOF_INT) || defined(INT_MAX) -#if SIZEOF_INT == 2 || INT_MAX == 0x7fff +#if defined(SIZEOF_INT) +#if SIZEOF_INT == 2 msgpack_pack_real_int16(x, d); -#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff +#elif SIZEOF_INT == 4 msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif + +#elif defined(INT_MAX) +#if INT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif INT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif + #else if(sizeof(int) == 2) { msgpack_pack_real_int16(x, d); @@ -419,14 +439,24 @@ if(sizeof(int) == 2) { msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d) { -#if defined(SIZEOF_LONG) || defined(LONG_MAX) -#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL +#if defined(SIZEOF_LONG) +#if SIZEOF_LONG == 2 msgpack_pack_real_int16(x, d); -#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL +#elif SIZEOF_LONG == 4 msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif + +#elif defined(LONG_MAX) +#if LONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif LONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif + #else if(sizeof(long) == 2) { msgpack_pack_real_int16(x, d); @@ -440,14 +470,24 @@ if(sizeof(long) == 2) { msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d) { -#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX) -#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL +#if defined(SIZEOF_LONG_LONG) +#if SIZEOF_LONG_LONG == 2 msgpack_pack_real_int16(x, d); -#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL +#elif SIZEOF_LONG_LONG == 4 msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif + +#elif defined(LLONG_MAX) +#if LLONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif LLONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif + #else if(sizeof(long long) == 2) { msgpack_pack_real_int16(x, d); @@ -461,14 +501,24 @@ if(sizeof(long long) == 2) { msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d) { -#if defined(SIZEOF_SHORT) || defined(USHRT_MAX) -#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU +#if defined(SIZEOF_SHORT) +#if SIZEOF_SHORT == 2 msgpack_pack_real_uint16(x, d); -#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU +#elif SIZEOF_SHORT == 4 msgpack_pack_real_uint32(x, d); #else msgpack_pack_real_uint64(x, d); #endif + +#elif defined(USHRT_MAX) +#if USHRT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif USHRT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif + #else if(sizeof(unsigned short) == 2) { msgpack_pack_real_uint16(x, d); @@ -482,14 +532,24 @@ if(sizeof(unsigned short) == 2) { msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d) { -#if defined(SIZEOF_INT) || defined(UINT_MAX) -#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU +#if defined(SIZEOF_INT) +#if SIZEOF_INT == 2 msgpack_pack_real_uint16(x, d); -#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU +#elif SIZEOF_INT == 4 msgpack_pack_real_uint32(x, d); #else msgpack_pack_real_uint64(x, d); #endif + +#elif defined(UINT_MAX) +#if UINT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif UINT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif + #else if(sizeof(unsigned int) == 2) { msgpack_pack_real_uint16(x, d); @@ -503,14 +563,24 @@ if(sizeof(unsigned int) == 2) { msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d) { -#if defined(SIZEOF_LONG) || defined(ULONG_MAX) -#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL +#if defined(SIZEOF_LONG) +#if SIZEOF_LONG == 2 msgpack_pack_real_uint16(x, d); -#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL +#elif SIZEOF_LONG == 4 msgpack_pack_real_uint32(x, d); #else msgpack_pack_real_uint64(x, d); #endif + +#elif defined(ULONG_MAX) +#if ULONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif ULONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif + #else if(sizeof(unsigned long) == 2) { msgpack_pack_real_uint16(x, d); @@ -524,14 +594,24 @@ if(sizeof(unsigned long) == 2) { msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d) { -#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX) -#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL +#if defined(SIZEOF_LONG_LONG) +#if SIZEOF_LONG_LONG == 2 msgpack_pack_real_uint16(x, d); -#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL +#elif SIZEOF_LONG_LONG == 4 msgpack_pack_real_uint32(x, d); #else msgpack_pack_real_uint64(x, d); #endif + +#elif defined(ULLONG_MAX) +#if ULLONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif ULLONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif + #else if(sizeof(unsigned long long) == 2) { msgpack_pack_real_uint16(x, d); From 42cb39a9775c2fb67071505091c9ef7afcdbf806 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 29 Apr 2010 07:01:16 +0900 Subject: [PATCH 0441/1172] Add COPYING file to python package. --- COPYING | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 COPYING diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..3cb1f1e --- /dev/null +++ b/COPYING @@ -0,0 +1,14 @@ +Copyright (C) 2008-2010 KLab Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + From f77d76a320547dc8ae2dde78b37f4119c9a2caf0 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 29 Apr 2010 07:01:16 +0900 Subject: [PATCH 0442/1172] Add COPYING file to python package. --- python/COPYING | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 python/COPYING diff --git a/python/COPYING b/python/COPYING new file mode 100644 index 0000000..3cb1f1e --- /dev/null +++ b/python/COPYING @@ -0,0 +1,14 @@ +Copyright (C) 2008-2010 KLab Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + From 316a0fa7b921dad15fe9b47a31e367b0218c0f38 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 29 Apr 2010 07:08:41 +0900 Subject: [PATCH 0443/1172] Add download url. --- MANIFEST.in | 1 + setup.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 6841ffe..cbc9ada 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include setup.py +include COPYING recursive-include msgpack *.h *.c *.pyx diff --git a/setup.py b/setup.py index 66cf27e..2feaff9 100755 --- a/setup.py +++ b/setup.py @@ -29,6 +29,10 @@ if have_cython: else: sources = ['msgpack/_msgpack.c'] + for f in sources: + if not os.path.exists(f): + raise ImportError("Building msgpack from VCS needs Cython. Install Cython or use sdist package.") + Sdist = sdist msgpack_mod = Extension('msgpack._msgpack', @@ -60,6 +64,7 @@ setup(name='msgpack', description=desc, long_description=long_desc, url="http://msgpack.sourceforge.jp/", + download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', From dda3d24bca8c00f58598523c60902c89581c8da2 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 29 Apr 2010 07:08:41 +0900 Subject: [PATCH 0444/1172] Add download url. --- python/MANIFEST.in | 1 + python/setup.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/python/MANIFEST.in b/python/MANIFEST.in index 6841ffe..cbc9ada 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -1,2 +1,3 @@ include setup.py +include COPYING recursive-include msgpack *.h *.c *.pyx diff --git a/python/setup.py b/python/setup.py index 66cf27e..2feaff9 100755 --- a/python/setup.py +++ b/python/setup.py @@ -29,6 +29,10 @@ if have_cython: else: sources = ['msgpack/_msgpack.c'] + for f in sources: + if not os.path.exists(f): + raise ImportError("Building msgpack from VCS needs Cython. Install Cython or use sdist package.") + Sdist = sdist msgpack_mod = Extension('msgpack._msgpack', @@ -60,6 +64,7 @@ setup(name='msgpack', description=desc, long_description=long_desc, url="http://msgpack.sourceforge.jp/", + download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', From 4f41080b942657b8bca4a3befe00eba0c80c6ce7 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 29 Apr 2010 07:52:32 +0900 Subject: [PATCH 0445/1172] Release msgpack-python 0.1.3 --- setup.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 2feaff9..27dbc48 100755 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ except ImportError: from distutils.command.build_ext import build_ext have_cython = False -version = '0.2.0dev' +version = '0.1.3' # take care of extension modules. if have_cython: @@ -42,11 +42,7 @@ del sources desc = 'MessagePack (de)serializer.' -long_desc = desc + """ - -MessagePack_ (de)serializer for Python. - -.. _MessagePack: http://msgpack.sourceforge.jp/ +long_desc = """MessagePack (de)serializer for Python. What's MessagePack? (from http://msgpack.sourceforge.jp/) @@ -54,7 +50,7 @@ What's MessagePack? (from http://msgpack.sourceforge.jp/) focused on high performance. It is like JSON, but very fast and small. """ -setup(name='msgpack', +setup(name='msgpack-python', author='INADA Naoki', author_email='songofacandy@gmail.com', version=version, @@ -63,7 +59,7 @@ setup(name='msgpack', packages=['msgpack'], description=desc, long_description=long_desc, - url="http://msgpack.sourceforge.jp/", + url='http://msgpack.sourceforge.jp/', download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ 'Development Status :: 4 - Beta', From 833ee6484c731d0419a9cbd122a499f2ae77ae6b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 29 Apr 2010 07:52:32 +0900 Subject: [PATCH 0446/1172] Release msgpack-python 0.1.3 --- python/setup.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/python/setup.py b/python/setup.py index 2feaff9..27dbc48 100755 --- a/python/setup.py +++ b/python/setup.py @@ -14,7 +14,7 @@ except ImportError: from distutils.command.build_ext import build_ext have_cython = False -version = '0.2.0dev' +version = '0.1.3' # take care of extension modules. if have_cython: @@ -42,11 +42,7 @@ del sources desc = 'MessagePack (de)serializer.' -long_desc = desc + """ - -MessagePack_ (de)serializer for Python. - -.. _MessagePack: http://msgpack.sourceforge.jp/ +long_desc = """MessagePack (de)serializer for Python. What's MessagePack? (from http://msgpack.sourceforge.jp/) @@ -54,7 +50,7 @@ What's MessagePack? (from http://msgpack.sourceforge.jp/) focused on high performance. It is like JSON, but very fast and small. """ -setup(name='msgpack', +setup(name='msgpack-python', author='INADA Naoki', author_email='songofacandy@gmail.com', version=version, @@ -63,7 +59,7 @@ setup(name='msgpack', packages=['msgpack'], description=desc, long_description=long_desc, - url="http://msgpack.sourceforge.jp/", + url='http://msgpack.sourceforge.jp/', download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ 'Development Status :: 4 - Beta', From 76cc80c25d10d1224f0f16c6a74b531bf452aede Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 29 Apr 2010 09:06:46 +0900 Subject: [PATCH 0447/1172] python: sourceforge.jp -> sourceforge.net --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 27dbc48..8d8a6f4 100755 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ del sources desc = 'MessagePack (de)serializer.' long_desc = """MessagePack (de)serializer for Python. -What's MessagePack? (from http://msgpack.sourceforge.jp/) +What's MessagePack? (from http://msgpack.sourceforge.net/) MessagePack is a binary-based efficient data interchange format that is focused on high performance. It is like JSON, but very fast and small. @@ -59,7 +59,7 @@ setup(name='msgpack-python', packages=['msgpack'], description=desc, long_description=long_desc, - url='http://msgpack.sourceforge.jp/', + url='http://msgpack.sourceforge.net/', download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ 'Development Status :: 4 - Beta', From 73ac804b4f7b8d6e1d762e5565fbf10b187c7beb Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 15 Jun 2010 17:51:24 +0900 Subject: [PATCH 0448/1172] Python: add "load(s)/dump(s)" alias for compatibility to simplejson/marshal/pickle. --- msgpack/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 797b29c..26bd2dd 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,3 +1,10 @@ # coding: utf-8 from _msgpack import * +# alias for compatibility to simplejson/marshal/pickle. +load = unpack +loads = unpackb + +dump = pack +dumps = packb + From f0e6d33f7965908988a8023ed734188d86f4f448 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 15 Jun 2010 18:06:58 +0900 Subject: [PATCH 0449/1172] Python: 0.1.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8d8a6f4..64e71ed 100755 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ except ImportError: from distutils.command.build_ext import build_ext have_cython = False -version = '0.1.3' +version = '0.1.4' # take care of extension modules. if have_cython: From 0dc8938dbfff67760834bd97150fee5fdb8e875c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 2 Sep 2010 01:29:57 +0900 Subject: [PATCH 0450/1172] python: Support Python3. --- msgpack/__init__.py | 2 +- msgpack/_msgpack.pyx | 37 +++++++++++++++++++------------------ msgpack/unpack.h | 2 +- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 797b29c..9593714 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,3 +1,3 @@ # coding: utf-8 -from _msgpack import * +from msgpack._msgpack import * diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 61ae36b..6a0b1a5 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -1,26 +1,24 @@ # coding: utf-8 -import cStringIO - cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject - cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + cdef object PyBytes_FromStringAndSize(const_char_ptr b, Py_ssize_t len) cdef PyObject* Py_True cdef PyObject* Py_False - cdef char* PyString_AsString(object o) cdef long long PyLong_AsLongLong(object o) cdef unsigned long long PyLong_AsUnsignedLongLong(object o) - cdef int PyMapping_Check(object o) - cdef int PySequence_Check(object o) - cdef int PyLong_Check(object o) - cdef int PyInt_Check(object o) - cdef int PyFloat_Check(object o) - cdef int PyString_Check(object o) - cdef int PyUnicode_Check(object o) + cdef bint PyBool_Check(object o) + cdef bint PyMapping_Check(object o) + cdef bint PySequence_Check(object o) + cdef bint PyLong_Check(object o) + cdef bint PyInt_Check(object o) + cdef bint PyFloat_Check(object o) + cdef bint PyBytes_Check(object o) + cdef bint PyUnicode_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -81,10 +79,12 @@ cdef class Packer(object): if o is None: ret = msgpack_pack_nil(&self.pk) - elif o == Py_True: - ret = msgpack_pack_true(&self.pk) - elif o == Py_False: - ret = msgpack_pack_false(&self.pk) + #elif PyBool_Check(o): + elif isinstance(o, bool): + if o: + ret = msgpack_pack_true(&self.pk) + else: + ret = msgpack_pack_false(&self.pk) elif PyLong_Check(o): if o > 0: ullval = PyLong_AsUnsignedLongLong(o) @@ -98,7 +98,7 @@ cdef class Packer(object): elif PyFloat_Check(o): fval = o ret = msgpack_pack_double(&self.pk, fval) - elif PyString_Check(o): + elif PyBytes_Check(o): rawval = o ret = msgpack_pack_raw(&self.pk, len(o)) if ret == 0: @@ -133,7 +133,7 @@ cdef class Packer(object): ret = self.__pack(obj) if ret: raise TypeError - buf = PyString_FromStringAndSize(self.pk.buf, self.pk.length) + buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 return buf @@ -262,10 +262,11 @@ cdef class Unpacker(object): cdef char* buf = self.buf cdef Py_ssize_t tail = self.buf_tail cdef Py_ssize_t l + cdef bytes b for b in self.waiting_bytes: l = len(b) - memcpy(buf + tail, PyString_AsString(b), l) + memcpy(buf + tail, (b), l) tail += l self.buf_tail = tail del self.waiting_bytes[:] diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 61a3786..9eb8ce7 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -175,7 +175,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { PyObject *py; - py = PyString_FromStringAndSize(p, l); + py = PyBytes_FromStringAndSize(p, l); if (!py) return -1; *o = py; From 1e8eeb8ebed50e8d39c69be653df06a10730631e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 2 Sep 2010 02:02:47 +0900 Subject: [PATCH 0451/1172] python: Fix Unpacker.feed doesn't accept bytes on Python3. --- msgpack/_msgpack.pyx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 6a0b1a5..85d717e 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -253,9 +253,7 @@ cdef class Unpacker(object): template_init(&self.ctx) self.ctx.user.use_list = use_list - def feed(self, next_bytes): - if not isinstance(next_bytes, str): - raise ValueError, "Argument must be bytes object" + def feed(self, bytes next_bytes): self.waiting_bytes.append(next_bytes) cdef append_buffer(self): From 039542ebcb8ca923c4414a414ecd62df43ff3f24 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 2 Sep 2010 02:16:28 +0900 Subject: [PATCH 0452/1172] python: Add test for python3 and fix found problems. --- msgpack/_msgpack.pyx | 20 ++++---- test3/test_case.py | 102 ++++++++++++++++++++++++++++++++++++++++ test3/test_except.py | 14 ++++++ test3/test_format.py | 75 +++++++++++++++++++++++++++++ test3/test_pack.py | 28 +++++++++++ test3/test_sequnpack.py | 36 ++++++++++++++ 6 files changed, 266 insertions(+), 9 deletions(-) create mode 100644 test3/test_case.py create mode 100644 test3/test_except.py create mode 100644 test3/test_format.py create mode 100644 test3/test_pack.py create mode 100644 test3/test_sequnpack.py diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 85d717e..c887127 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -12,7 +12,7 @@ cdef extern from "Python.h": cdef unsigned long long PyLong_AsUnsignedLongLong(object o) cdef bint PyBool_Check(object o) - cdef bint PyMapping_Check(object o) + cdef bint PyDict_Check(object o) cdef bint PySequence_Check(object o) cdef bint PyLong_Check(object o) cdef bint PyInt_Check(object o) @@ -69,13 +69,14 @@ cdef class Packer(object): def __dealloc__(self): free(self.pk.buf); - cdef int __pack(self, object o) except -1: + cdef int _pack(self, object o) except -1: cdef long long llval cdef unsigned long long ullval cdef long longval cdef double fval cdef char* rawval cdef int ret + cdef dict d if o is None: ret = msgpack_pack_nil(&self.pk) @@ -109,19 +110,20 @@ cdef class Packer(object): ret = msgpack_pack_raw(&self.pk, len(o)) if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) - elif PyMapping_Check(o): - ret = msgpack_pack_map(&self.pk, len(o)) + elif PyDict_Check(o): + d = o + ret = msgpack_pack_map(&self.pk, len(d)) if ret == 0: - for k,v in o.iteritems(): - ret = self.__pack(k) + for k,v in d.items(): + ret = self._pack(k) if ret != 0: break - ret = self.__pack(v) + ret = self._pack(v) if ret != 0: break elif PySequence_Check(o): ret = msgpack_pack_array(&self.pk, len(o)) if ret == 0: for v in o: - ret = self.__pack(v) + ret = self._pack(v) if ret != 0: break else: # TODO: Serialize with defalt() like simplejson. @@ -130,7 +132,7 @@ cdef class Packer(object): def pack(self, object obj): cdef int ret - ret = self.__pack(obj) + ret = self._pack(obj) if ret: raise TypeError buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) diff --git a/test3/test_case.py b/test3/test_case.py new file mode 100644 index 0000000..53dfcaf --- /dev/null +++ b/test3/test_case.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * +from msgpack import packs, unpacks + + +def check(length, obj): + v = packs(obj) + assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) + assert_equal(unpacks(v), obj) + +def test_1(): + for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1, + -((1<<5)-1), -(1<<5)]: + check(1, o) + +def test_2(): + for o in [1 << 7, (1 << 8) - 1, + -((1<<5)+1), -(1<<7) + ]: + check(2, o) + +def test_3(): + for o in [1 << 8, (1 << 16) - 1, + -((1<<7)+1), -(1<<15)]: + check(3, o) + +def test_5(): + for o in [1 << 16, (1 << 32) - 1, + -((1<<15)+1), -(1<<31)]: + check(5, o) + +def test_9(): + for o in [1 << 32, (1 << 64) - 1, + -((1<<31)+1), -(1<<63), + 1.0, 0.1, -0.1, -1.0]: + check(9, o) + + +def check_raw(overhead, num): + check(num + overhead, b" " * num) + +def test_fixraw(): + check_raw(1, 0) + check_raw(1, (1<<5) - 1) + +def test_raw16(): + check_raw(3, 1<<5) + check_raw(3, (1<<16) - 1) + +def test_raw32(): + check_raw(5, 1<<16) + + +def check_array(overhead, num): + check(num + overhead, (None,) * num) + +def test_fixarray(): + check_array(1, 0) + check_array(1, (1 << 4) - 1) + +def test_array16(): + check_array(3, 1 << 4) + check_array(3, (1<<16)-1) + +def test_array32(): + check_array(5, (1<<16)) + + +def match(obj, buf): + assert_equal(packs(obj), buf) + assert_equal(unpacks(buf), obj) + +def test_match(): + cases = [ + (None, b'\xc0'), + (False, b'\xc2'), + (True, b'\xc3'), + (0, b'\x00'), + (127, b'\x7f'), + (128, b'\xcc\x80'), + (256, b'\xcd\x01\x00'), + (-1, b'\xff'), + (-33, b'\xd0\xdf'), + (-129, b'\xd1\xff\x7f'), + ({1:1}, b'\x81\x01\x01'), + (1.0, b"\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"), + ((), b'\x90'), + (tuple(range(15)),b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), + (tuple(range(16)),b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), + ({}, b'\x80'), + (dict([(x,x) for x in range(15)]), b'\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'), + (dict([(x,x) for x in range(16)]), b'\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'), + ] + + for v, p in cases: + match(v, p) + +if __name__ == '__main__': + main() diff --git a/test3/test_except.py b/test3/test_except.py new file mode 100644 index 0000000..574728f --- /dev/null +++ b/test3/test_except.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose.tools import * +from msgpack import packs, unpacks + +import datetime + +def test_raise_on_find_unsupported_value(): + assert_raises(TypeError, packs, datetime.datetime.now()) + +if __name__ == '__main__': + from nose import main + main() diff --git a/test3/test_format.py b/test3/test_format.py new file mode 100644 index 0000000..022e680 --- /dev/null +++ b/test3/test_format.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * +from msgpack import unpacks + +def check(src, should): + assert_equal(unpacks(src), should) + +def testSimpleValue(): + check(b"\x93\xc0\xc2\xc3", + (None, False, True,)) + +def testFixnum(): + check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", + ((0,64,127,), (-32,-16,-1,),) + ) + +def testFixArray(): + check(b"\x92\x90\x91\x91\xc0", + ((),((None,),),), + ) + +def testFixRaw(): + check(b"\x94\xa0\xa1a\xa2bc\xa3def", + (b"", b"a", b"bc", b"def",), + ) + +def testFixMap(): + check( + b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", + {False: {None: None}, True:{None:{}}}, + ) + +def testUnsignedInt(): + check( + b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" + b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" + b"\xce\xff\xff\xff\xff", + (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,), + ) + +def testSignedInt(): + check(b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" + b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" + b"\xd2\xff\xff\xff\xff", + (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,)) + +def testRaw(): + check(b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" + b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", + (b"", b"a", b"ab", b"", b"a", b"ab")) + +def testArray(): + check(b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" + b"\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02" + b"\xc2\xc3", + ((), (None,), (False,True), (), (None,), (False,True)) + ) + +def testMap(): + check( + b"\x96" + b"\xde\x00\x00" + b"\xde\x00\x01\xc0\xc2" + b"\xde\x00\x02\xc0\xc2\xc3\xc2" + b"\xdf\x00\x00\x00\x00" + b"\xdf\x00\x00\x00\x01\xc0\xc2" + b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", + ({}, {None: False}, {True: False, None: False}, {}, + {None: False}, {True: False, None: False})) + +if __name__ == '__main__': + main() diff --git a/test3/test_pack.py b/test3/test_pack.py new file mode 100644 index 0000000..c861704 --- /dev/null +++ b/test3/test_pack.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * + +from msgpack import packs, unpacks + +def check(data): + re = unpacks(packs(data)) + assert_equal(re, data) + +def testPack(): + test_data = [ + 0, 1, 127, 128, 255, 256, 65535, 65536, + -1, -32, -33, -128, -129, -32768, -32769, + 1.0, + b"", b"a", b"a"*31, b"a"*32, + None, True, False, + (), ((),), ((), None,), + {None: 0}, + (1<<23), + ] + for td in test_data: + check(td) + +if __name__ == '__main__': + main() diff --git a/test3/test_sequnpack.py b/test3/test_sequnpack.py new file mode 100644 index 0000000..5fd377c --- /dev/null +++ b/test3/test_sequnpack.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# coding: utf-8 + + + +from msgpack import Unpacker + +def test_foobar(): + unpacker = Unpacker(read_size=3) + unpacker.feed(b'foobar') + assert unpacker.unpack() == ord(b'f') + assert unpacker.unpack() == ord(b'o') + assert unpacker.unpack() == ord(b'o') + assert unpacker.unpack() == ord(b'b') + assert unpacker.unpack() == ord(b'a') + assert unpacker.unpack() == ord(b'r') + try: + o = unpacker.unpack() + print(("Oops!", o)) + assert 0 + except StopIteration: + assert 1 + else: + assert 0 + unpacker.feed(b'foo') + unpacker.feed(b'bar') + + k = 0 + for o, e in zip(unpacker, b'foobarbaz'): + assert o == e + k += 1 + assert k == len(b'foobar') + +if __name__ == '__main__': + test_foobar() + From 50ffd2c489fdab81da2d994794b9582e0cadf5f4 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 2 Sep 2010 09:54:38 +0900 Subject: [PATCH 0453/1172] Add msgpack.version as version tuple. --- .gitignore | 1 + Makefile | 7 ++++++- msgpack/__init__.py | 1 + setup.py | 8 ++++++-- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 430c633..8531de3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ build/* dist/* *.pyc *.pyo +msgpack/__version__.py diff --git a/Makefile b/Makefile index e06794d..245c09c 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,12 @@ +.PHONY: test all python3 + all: python setup.py build_ext -i -f python setup.py build sdist -.PHONY: test +python3: + python3 setup.py build_ext -i -f + python3 setup.py build sdist + test: nosetests test diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 86786a2..cdf045f 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,4 +1,5 @@ # coding: utf-8 +from msgpack.__version__ import * from msgpack._msgpack import * # alias for compatibility to simplejson/marshal/pickle. diff --git a/setup.py b/setup.py index 64e71ed..c79c148 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # coding: utf-8 +version = (0, 1, 5, 'dev') import os from glob import glob @@ -14,7 +15,10 @@ except ImportError: from distutils.command.build_ext import build_ext have_cython = False -version = '0.1.4' +# make msgpack/__verison__.py +f = open('msgpack/__version__.py', 'w') +f.write("version = %r\n" % (version,)) +f.close() # take care of extension modules. if have_cython: @@ -53,7 +57,7 @@ What's MessagePack? (from http://msgpack.sourceforge.net/) setup(name='msgpack-python', author='INADA Naoki', author_email='songofacandy@gmail.com', - version=version, + version=''.join(str(x) for x in version), cmdclass={'build_ext': build_ext, 'sdist': Sdist}, ext_modules=[msgpack_mod], packages=['msgpack'], From 742ca5c341ac1d51680d662b8833cb2a6cf9064d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 2 Sep 2010 09:58:50 +0900 Subject: [PATCH 0454/1172] python: vesion 0.1.5 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c79c148..67ff74c 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 1, 5, 'dev') +version = (0, 1, 5, 'final') import os from glob import glob From b68f61cae998c2d690501393563ef6a29ac35154 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 2 Sep 2010 10:10:34 +0900 Subject: [PATCH 0455/1172] python: Release 0.1.6 - Fix wrong version string. --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 67ff74c..d079e3e 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 1, 5, 'final') +version = (0, 1, 6, 'final') import os from glob import glob @@ -19,6 +19,9 @@ except ImportError: f = open('msgpack/__version__.py', 'w') f.write("version = %r\n" % (version,)) f.close() +version_str = '.'.join(str(x) for x in version[:3]) +if len(version) > 3 and version[3] != 'final': + version_str += version[3] # take care of extension modules. if have_cython: @@ -57,7 +60,7 @@ What's MessagePack? (from http://msgpack.sourceforge.net/) setup(name='msgpack-python', author='INADA Naoki', author_email='songofacandy@gmail.com', - version=''.join(str(x) for x in version), + version=version_str, cmdclass={'build_ext': build_ext, 'sdist': Sdist}, ext_modules=[msgpack_mod], packages=['msgpack'], From ff5709718bd8516e895cc461f4db57ff601abc49 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 2 Sep 2010 10:13:49 +0900 Subject: [PATCH 0456/1172] python: Add python3 category. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index d079e3e..ac7ece5 100755 --- a/setup.py +++ b/setup.py @@ -69,6 +69,7 @@ setup(name='msgpack-python', url='http://msgpack.sourceforge.net/', download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ + 'Programming Language :: Python :: 3', 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', From 9db35d88316dc652d371de2f1fd3a82e9d285f70 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 7 Oct 2010 03:02:02 +0900 Subject: [PATCH 0457/1172] Use PyUnicode_AsUTF8String() instead of o.encode('utf-8'). --- msgpack/_msgpack.pyx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index c887127..66869c8 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -7,6 +7,7 @@ cdef extern from "Python.h": cdef object PyBytes_FromStringAndSize(const_char_ptr b, Py_ssize_t len) cdef PyObject* Py_True cdef PyObject* Py_False + cdef object PyUnicode_AsUTF8String(object) cdef long long PyLong_AsLongLong(object o) cdef unsigned long long PyLong_AsUnsignedLongLong(object o) @@ -105,7 +106,7 @@ cdef class Packer(object): if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyUnicode_Check(o): - o = o.encode('utf-8') + o = PyUnicode_AsUTF8String(o) rawval = o ret = msgpack_pack_raw(&self.pk, len(o)) if ret == 0: @@ -169,7 +170,7 @@ cdef extern from "unpack.h": object template_data(template_context* ctx) -def unpackb(object packed_bytes): +def unpackb(bytes packed_bytes): """Unpack packed_bytes to object. Returns an unpacked object.""" cdef const_char_ptr p = packed_bytes cdef template_context ctx @@ -232,7 +233,7 @@ cdef class Unpacker(object): cdef object file_like cdef int read_size cdef object waiting_bytes - cdef int use_list + cdef bint use_list def __cinit__(self): self.buf = NULL @@ -241,7 +242,7 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=0, use_list=0): + def __init__(self, file_like=None, int read_size=0, bint use_list=0): if read_size == 0: read_size = 1024*1024 self.use_list = use_list From 4e3c620d367a99eabf2e28d8a6fd371fa33b5dcf Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 7 Oct 2010 03:04:00 +0900 Subject: [PATCH 0458/1172] Fix testcase for unicode. --- test3/test_case.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test3/test_case.py b/test3/test_case.py index 53dfcaf..2f42316 100644 --- a/test3/test_case.py +++ b/test3/test_case.py @@ -98,5 +98,8 @@ def test_match(): for v, p in cases: match(v, p) +def test_unicode(): + assert_equal(b'foobar', unpacks(packs('foobar'))) + if __name__ == '__main__': main() From 367f15c247bf37beb37e2a81d1d707bc8ef2e085 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 7 Oct 2010 03:07:52 +0900 Subject: [PATCH 0459/1172] Add unicode testcase for Python2. --- test/test_case.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_case.py b/test/test_case.py index a08c6ce..1cbc494 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -98,5 +98,8 @@ def test_match(): for v, p in cases: match(v, p) +def test_unicode(): + assert_equal('foobar', unpacks(packs(u'foobar'))) + if __name__ == '__main__': main() From fa157082ac8db71e3312ca97fe1ceb7f56546fcb Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 26 Oct 2010 01:26:06 +0900 Subject: [PATCH 0460/1172] Add `object_hook` option to unpack and `default` option to pack. (see simplejson for how to use). --- msgpack/_msgpack.pyx | 56 ++++++++++++++++++++++++++++++++------- msgpack/unpack.h | 14 ++++++++++ msgpack/unpack_template.h | 1 + test/test_obj.py | 31 ++++++++++++++++++++++ 4 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 test/test_obj.py diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 66869c8..fb7f0c1 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -20,6 +20,9 @@ cdef extern from "Python.h": cdef bint PyFloat_Check(object o) cdef bint PyBytes_Check(object o) cdef bint PyUnicode_Check(object o) + cdef bint PyCallable_Check(object o) + cdef void Py_INCREF(object o) + cdef void Py_DECREF(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -60,6 +63,7 @@ cdef class Packer(object): astream.write(packer.pack(b)) """ cdef msgpack_packer pk + cdef object default def __cinit__(self): cdef int buf_size = 1024*1024 @@ -67,6 +71,12 @@ cdef class Packer(object): self.pk.buf_size = buf_size self.pk.length = 0 + def __init__(self, default=None): + if default is not None: + if not PyCallable_Check(default): + raise TypeError("default must be a callable.") + self.default = default + def __dealloc__(self): free(self.pk.buf); @@ -126,9 +136,18 @@ cdef class Packer(object): for v in o: ret = self._pack(v) if ret != 0: break + elif self.default is not None: + o = self.default(o) + d = o + ret = msgpack_pack_map(&self.pk, len(d)) + if ret == 0: + for k,v in d.items(): + ret = self._pack(k) + if ret != 0: break + ret = self._pack(v) + if ret != 0: break else: - # TODO: Serialize with defalt() like simplejson. - raise TypeError, "can't serialize %r" % (o,) + raise TypeError("can't serialize %r" % (o,)) return ret def pack(self, object obj): @@ -141,14 +160,14 @@ cdef class Packer(object): return buf -def pack(object o, object stream): +def pack(object o, object stream, default=None): """pack an object `o` and write it to stream).""" - packer = Packer() + packer = Packer(default) stream.write(packer.pack(o)) -def packb(object o): +def packb(object o, default=None): """pack o and return packed bytes.""" - packer = Packer() + packer = Packer(default=default) return packer.pack(o) packs = packb @@ -156,6 +175,7 @@ packs = packb cdef extern from "unpack.h": ctypedef struct msgpack_user: int use_list + PyObject* object_hook ctypedef struct template_context: msgpack_user user @@ -170,7 +190,7 @@ cdef extern from "unpack.h": object template_data(template_context* ctx) -def unpackb(bytes packed_bytes): +def unpackb(bytes packed_bytes, object object_hook=None): """Unpack packed_bytes to object. Returns an unpacked object.""" cdef const_char_ptr p = packed_bytes cdef template_context ctx @@ -178,7 +198,16 @@ def unpackb(bytes packed_bytes): cdef int ret template_init(&ctx) ctx.user.use_list = 0 + ctx.user.object_hook = NULL + if object_hook is not None: + if not PyCallable_Check(object_hook): + raise TypeError("object_hook must be a callable.") + Py_INCREF(object_hook) + ctx.user.object_hook = object_hook ret = template_execute(&ctx, p, len(packed_bytes), &off) + if object_hook is not None: + pass + #Py_DECREF(object_hook) if ret == 1: return template_data(&ctx) else: @@ -186,10 +215,10 @@ def unpackb(bytes packed_bytes): unpacks = unpackb -def unpack(object stream): +def unpack(object stream, object object_hook=None): """unpack an object from stream.""" packed = stream.read() - return unpackb(packed) + return unpackb(packed, object_hook=object_hook) cdef class UnpackIterator(object): cdef object unpacker @@ -234,6 +263,7 @@ cdef class Unpacker(object): cdef int read_size cdef object waiting_bytes cdef bint use_list + cdef object object_hook def __cinit__(self): self.buf = NULL @@ -242,7 +272,8 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=0, bint use_list=0): + def __init__(self, file_like=None, int read_size=0, bint use_list=0, + object object_hook=None): if read_size == 0: read_size = 1024*1024 self.use_list = use_list @@ -255,6 +286,11 @@ cdef class Unpacker(object): self.buf_tail = 0 template_init(&self.ctx) self.ctx.user.use_list = use_list + self.ctx.user.object_hook = NULL + if object_hook is not None: + if not PyCallable_Check(object_hook): + raise TypeError("object_hook must be a callable.") + self.ctx.user.object_hook = object_hook def feed(self, bytes next_bytes): self.waiting_bytes.append(next_bytes) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 9eb8ce7..e4c03bd 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -21,6 +21,7 @@ typedef struct unpack_user { int use_list; + PyObject *object_hook; } unpack_user; @@ -172,6 +173,19 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje return -1; } +//static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c) +int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c) +{ + if (u->object_hook) { + PyObject *arglist = Py_BuildValue("(O)", *c); + Py_INCREF(*c); + *c = PyEval_CallObject(u->object_hook, arglist); + Py_DECREF(arglist); + return 0; + } + return -1; +} + static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { PyObject *py; diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index ca6e1f3..1fdedd7 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -317,6 +317,7 @@ _push: case CT_MAP_VALUE: if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } if(--c->count == 0) { + msgpack_unpack_callback(_map_end)(user, &c->obj); obj = c->obj; --top; /*printf("stack pop %d\n", top);*/ diff --git a/test/test_obj.py b/test/test_obj.py new file mode 100644 index 0000000..64a6390 --- /dev/null +++ b/test/test_obj.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * + +from msgpack import packs, unpacks + +def _decode_complex(obj): + if '__complex__' in obj: + return complex(obj['real'], obj['imag']) + return obj + +def _encode_complex(obj): + if isinstance(obj, complex): + return {'__complex__': True, 'real': 1, 'imag': 2} + return obj + +def test_encode_hook(): + packed = packs([3, 1+2j], default=_encode_complex) + unpacked = unpacks(packed) + eq_(unpacked[1], {'__complex__': True, 'real': 1, 'imag': 2}) + +def test_decode_hook(): + packed = packs([3, {'__complex__': True, 'real': 1, 'imag': 2}]) + unpacked = unpacks(packed, object_hook=_decode_complex) + eq_(unpacked[1], 1+2j) + +if __name__ == '__main__': + #main() + test_decode_hook() From c355f224dc4293683a426e8620a07724ab0e75ac Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 26 Oct 2010 01:31:27 +0900 Subject: [PATCH 0461/1172] Add test for Python3. --- test3/test_obj.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 test3/test_obj.py diff --git a/test3/test_obj.py b/test3/test_obj.py new file mode 100644 index 0000000..972d1ca --- /dev/null +++ b/test3/test_obj.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * + +from msgpack import packs, unpacks + +def _decode_complex(obj): + if b'__complex__' in obj: + return complex(obj[b'real'], obj[b'imag']) + return obj + +def _encode_complex(obj): + if isinstance(obj, complex): + return {b'__complex__': True, b'real': 1, b'imag': 2} + return obj + +def test_encode_hook(): + packed = packs([3, 1+2j], default=_encode_complex) + unpacked = unpacks(packed) + eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2}) + +def test_decode_hook(): + packed = packs([3, {b'__complex__': True, b'real': 1, b'imag': 2}]) + unpacked = unpacks(packed, object_hook=_decode_complex) + eq_(unpacked[1], 1+2j) + +if __name__ == '__main__': + #main() + test_decode_hook() From 3980d381f7ad44b14483dc90c0f6f0a36804c290 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 26 Oct 2010 01:32:08 +0900 Subject: [PATCH 0462/1172] Remove unnecessary refcount manipulation. --- msgpack/_msgpack.pyx | 4 ---- msgpack/unpack.h | 1 - 2 files changed, 5 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index fb7f0c1..24e4f8b 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -202,12 +202,8 @@ def unpackb(bytes packed_bytes, object object_hook=None): if object_hook is not None: if not PyCallable_Check(object_hook): raise TypeError("object_hook must be a callable.") - Py_INCREF(object_hook) ctx.user.object_hook = object_hook ret = template_execute(&ctx, p, len(packed_bytes), &off) - if object_hook is not None: - pass - #Py_DECREF(object_hook) if ret == 1: return template_data(&ctx) else: diff --git a/msgpack/unpack.h b/msgpack/unpack.h index e4c03bd..404ee5a 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -178,7 +178,6 @@ int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c) { if (u->object_hook) { PyObject *arglist = Py_BuildValue("(O)", *c); - Py_INCREF(*c); *c = PyEval_CallObject(u->object_hook, arglist); Py_DECREF(arglist); return 0; From 0076d42a0ddeb3601f1a0c806c7874da2110a986 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 26 Oct 2010 01:49:00 +0900 Subject: [PATCH 0463/1172] Add check for recursion limit and default hook result. --- msgpack/_msgpack.pyx | 24 ++++++++++-------------- test/test_obj.py | 8 +++++++- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 24e4f8b..0abdd51 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -80,7 +80,7 @@ cdef class Packer(object): def __dealloc__(self): free(self.pk.buf); - cdef int _pack(self, object o) except -1: + cdef int _pack(self, object o, int nest_limit=511, default=None) except -1: cdef long long llval cdef unsigned long long ullval cdef long longval @@ -89,6 +89,9 @@ cdef class Packer(object): cdef int ret cdef dict d + if nest_limit < 0: + raise ValueError("Too deep.") + if o is None: ret = msgpack_pack_nil(&self.pk) #elif PyBool_Check(o): @@ -126,33 +129,26 @@ cdef class Packer(object): ret = msgpack_pack_map(&self.pk, len(d)) if ret == 0: for k,v in d.items(): - ret = self._pack(k) + ret = self._pack(k, nest_limit-1, default) if ret != 0: break - ret = self._pack(v) + ret = self._pack(v, nest_limit-1, default) if ret != 0: break elif PySequence_Check(o): ret = msgpack_pack_array(&self.pk, len(o)) if ret == 0: for v in o: - ret = self._pack(v) + ret = self._pack(v, nest_limit-1, default) if ret != 0: break - elif self.default is not None: + elif default is not None: o = self.default(o) - d = o - ret = msgpack_pack_map(&self.pk, len(d)) - if ret == 0: - for k,v in d.items(): - ret = self._pack(k) - if ret != 0: break - ret = self._pack(v) - if ret != 0: break + ret = self._pack(o, nest_limit) else: raise TypeError("can't serialize %r" % (o,)) return ret def pack(self, object obj): cdef int ret - ret = self._pack(obj) + ret = self._pack(obj, self.default) if ret: raise TypeError buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) diff --git a/test/test_obj.py b/test/test_obj.py index 64a6390..28edacb 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -26,6 +26,12 @@ def test_decode_hook(): unpacked = unpacks(packed, object_hook=_decode_complex) eq_(unpacked[1], 1+2j) +@raises(TypeError) +def test_bad_hook(): + packed = packs([3, 1+2j], default=lambda o: o) + unpacked = unpacks(packed) + if __name__ == '__main__': - #main() test_decode_hook() + test_encode_hook() + test_bad_hook() From 3d8978417a177c280b837d72054848e8dd4bc649 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 26 Oct 2010 02:09:52 +0900 Subject: [PATCH 0464/1172] Add list_hook option to unpacker. --- msgpack/_msgpack.pyx | 27 +++++++++++++++++++-------- msgpack/unpack.h | 17 +++++++++++++---- msgpack/unpack_template.h | 3 ++- test/test_obj.py | 9 +++++++++ 4 files changed, 43 insertions(+), 13 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 0abdd51..e9d6c7b 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -52,6 +52,7 @@ cdef extern from "pack.h": int msgpack_pack_raw(msgpack_packer* pk, size_t l) int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) +cdef int DEFAULT_RECURSE_LIMIT=511 cdef class Packer(object): """MessagePack Packer @@ -80,7 +81,8 @@ cdef class Packer(object): def __dealloc__(self): free(self.pk.buf); - cdef int _pack(self, object o, int nest_limit=511, default=None) except -1: + cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT, + default=None) except -1: cdef long long llval cdef unsigned long long ullval cdef long longval @@ -148,7 +150,7 @@ cdef class Packer(object): def pack(self, object obj): cdef int ret - ret = self._pack(obj, self.default) + ret = self._pack(obj, DEFAULT_RECURSE_LIMIT, self.default) if ret: raise TypeError buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) @@ -172,6 +174,7 @@ cdef extern from "unpack.h": ctypedef struct msgpack_user: int use_list PyObject* object_hook + PyObject* list_hook ctypedef struct template_context: msgpack_user user @@ -186,7 +189,7 @@ cdef extern from "unpack.h": object template_data(template_context* ctx) -def unpackb(bytes packed_bytes, object object_hook=None): +def unpackb(bytes packed_bytes, object object_hook=None, object list_hook=None): """Unpack packed_bytes to object. Returns an unpacked object.""" cdef const_char_ptr p = packed_bytes cdef template_context ctx @@ -194,11 +197,15 @@ def unpackb(bytes packed_bytes, object object_hook=None): cdef int ret template_init(&ctx) ctx.user.use_list = 0 - ctx.user.object_hook = NULL + ctx.user.object_hook = ctx.user.list_hook = NULL if object_hook is not None: if not PyCallable_Check(object_hook): raise TypeError("object_hook must be a callable.") ctx.user.object_hook = object_hook + if list_hook is not None: + if not PyCallable_Check(list_hook): + raise TypeError("list_hook must be a callable.") + ctx.user.list_hook = list_hook ret = template_execute(&ctx, p, len(packed_bytes), &off) if ret == 1: return template_data(&ctx) @@ -207,10 +214,10 @@ def unpackb(bytes packed_bytes, object object_hook=None): unpacks = unpackb -def unpack(object stream, object object_hook=None): +def unpack(object stream, object object_hook=None, object list_hook=None): """unpack an object from stream.""" packed = stream.read() - return unpackb(packed, object_hook=object_hook) + return unpackb(packed, object_hook=object_hook, list_hook=list_hook) cdef class UnpackIterator(object): cdef object unpacker @@ -265,7 +272,7 @@ cdef class Unpacker(object): free(self.buf); def __init__(self, file_like=None, int read_size=0, bint use_list=0, - object object_hook=None): + object object_hook=None, object list_hook=None): if read_size == 0: read_size = 1024*1024 self.use_list = use_list @@ -278,11 +285,15 @@ cdef class Unpacker(object): self.buf_tail = 0 template_init(&self.ctx) self.ctx.user.use_list = use_list - self.ctx.user.object_hook = NULL + self.ctx.user.object_hook = self.ctx.user.list_hook = NULL if object_hook is not None: if not PyCallable_Check(object_hook): raise TypeError("object_hook must be a callable.") self.ctx.user.object_hook = object_hook + if list_hook is not None: + if not PyCallable_Check(list_hook): + raise TypeError("object_hook must be a callable.") + self.ctx.user.list_hook = list_hook def feed(self, bytes next_bytes): self.waiting_bytes.append(next_bytes) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 404ee5a..453ec2b 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -22,6 +22,7 @@ typedef struct unpack_user { int use_list; PyObject *object_hook; + PyObject *list_hook; } unpack_user; @@ -154,6 +155,16 @@ static inline int template_callback_array_item(unpack_user* u, unsigned int curr return 0; } +static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_object* c) +{ + if (u->list_hook) { + PyObject *arglist = Py_BuildValue("(O)", *c); + *c = PyEval_CallObject(u->list_hook, arglist); + Py_DECREF(arglist); + } + return 0; +} + static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { PyObject *p = PyDict_New(); @@ -173,16 +184,14 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje return -1; } -//static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c) -int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c) +static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c) { if (u->object_hook) { PyObject *arglist = Py_BuildValue("(O)", *c); *c = PyEval_CallObject(u->object_hook, arglist); Py_DECREF(arglist); - return 0; } - return -1; + return 0; } static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 1fdedd7..7a2288f 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -304,6 +304,7 @@ _push: case CT_ARRAY_ITEM: if(msgpack_unpack_callback(_array_item)(user, c->curr, &c->obj, obj) < 0) { goto _failed; } if(++c->curr == c->count) { + msgpack_unpack_callback(_array_end)(user, &c->obj); obj = c->obj; --top; /*printf("stack pop %d\n", top);*/ @@ -317,7 +318,7 @@ _push: case CT_MAP_VALUE: if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } if(--c->count == 0) { - msgpack_unpack_callback(_map_end)(user, &c->obj); + msgpack_unpack_callback(_map_end)(user, &c->obj); obj = c->obj; --top; /*printf("stack pop %d\n", top);*/ diff --git a/test/test_obj.py b/test/test_obj.py index 28edacb..bc85736 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -31,7 +31,16 @@ def test_bad_hook(): packed = packs([3, 1+2j], default=lambda o: o) unpacked = unpacks(packed) +def _arr_to_str(arr): + return ''.join(str(c) for c in arr) + +def test_array_hook(): + packed = packs([1,2,3]) + unpacked = unpacks(packed, list_hook=_arr_to_str) + eq_(unpacked, '123') + if __name__ == '__main__': test_decode_hook() test_encode_hook() test_bad_hook() + test_array_hook() From 8d3ca93be9639f4a1066b39a8ee37f626dcb8c3c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 2 Nov 2010 14:02:10 +0900 Subject: [PATCH 0465/1172] python: Add msgpack/_msgpack.c to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8531de3..35b00cc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ dist/* *.pyc *.pyo msgpack/__version__.py +msgpack/_msgpack.c From 3903979a84c4d450acc55faceb3f816ff7a32398 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 2 Nov 2010 14:09:50 +0900 Subject: [PATCH 0466/1172] python: Add test for unpacking buffer object. --- test/test_buffer.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 test/test_buffer.py diff --git a/test/test_buffer.py b/test/test_buffer.py new file mode 100644 index 0000000..436b554 --- /dev/null +++ b/test/test_buffer.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * +from msgpack import packb, unpackb + +def test_unpack_buffer(): + from array import array + buf = array('b') + buf.fromstring(packb(['foo', 'bar'])) + obj = unpackb(buf) + assert_equal(['foo', 'bar'], obj) + +if __name__ == '__main__': + main() From a09c85ff9c846384cf681fa955ce931010995055 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 3 Nov 2010 03:11:00 +0900 Subject: [PATCH 0467/1172] python: Support old buffer protocol when unpack. (experimental) --- msgpack/_msgpack.pyx | 156 ++++++++++++++--------------------------- test/test_buffer.py | 6 +- test/test_sequnpack.py | 16 ++--- 3 files changed, 65 insertions(+), 113 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index e9d6c7b..5394055 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -1,37 +1,14 @@ # coding: utf-8 +from cpython cimport * cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" + ctypedef char* const_void_ptr "const void*" ctypedef struct PyObject + cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1 - cdef object PyBytes_FromStringAndSize(const_char_ptr b, Py_ssize_t len) - cdef PyObject* Py_True - cdef PyObject* Py_False - cdef object PyUnicode_AsUTF8String(object) - - cdef long long PyLong_AsLongLong(object o) - cdef unsigned long long PyLong_AsUnsignedLongLong(object o) - - cdef bint PyBool_Check(object o) - cdef bint PyDict_Check(object o) - cdef bint PySequence_Check(object o) - cdef bint PyLong_Check(object o) - cdef bint PyInt_Check(object o) - cdef bint PyFloat_Check(object o) - cdef bint PyBytes_Check(object o) - cdef bint PyUnicode_Check(object o) - cdef bint PyCallable_Check(object o) - cdef void Py_INCREF(object o) - cdef void Py_DECREF(object o) - -cdef extern from "stdlib.h": - void* malloc(size_t) - void* realloc(void*, size_t) - void free(void*) - -cdef extern from "string.h": - void* memcpy(char* dst, char* src, size_t size) - void* memmove(char* dst, char* src, size_t size) +from libc.stdlib cimport * +from libc.string cimport * cdef extern from "pack.h": struct msgpack_packer: @@ -104,10 +81,10 @@ cdef class Packer(object): ret = msgpack_pack_false(&self.pk) elif PyLong_Check(o): if o > 0: - ullval = PyLong_AsUnsignedLongLong(o) + ullval = o ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) else: - llval = PyLong_AsLongLong(o) + llval = o ret = msgpack_pack_long_long(&self.pk, llval) elif PyInt_Check(o): longval = o @@ -160,7 +137,7 @@ cdef class Packer(object): def pack(object o, object stream, default=None): """pack an object `o` and write it to stream).""" - packer = Packer(default) + packer = Packer(default=default) stream.write(packer.pack(o)) def packb(object o, default=None): @@ -184,17 +161,21 @@ cdef extern from "unpack.h": PyObject* key int template_execute(template_context* ctx, const_char_ptr data, - size_t len, size_t* off) + size_t len, size_t* off) void template_init(template_context* ctx) object template_data(template_context* ctx) -def unpackb(bytes packed_bytes, object object_hook=None, object list_hook=None): +def unpackb(object packed, object object_hook=None, object list_hook=None): """Unpack packed_bytes to object. Returns an unpacked object.""" - cdef const_char_ptr p = packed_bytes cdef template_context ctx cdef size_t off = 0 cdef int ret + + cdef char* buf + cdef Py_ssize_t buf_len + PyObject_AsReadBuffer(packed, &buf, &buf_len) + template_init(&ctx) ctx.user.use_list = 0 ctx.user.object_hook = ctx.user.list_hook = NULL @@ -206,7 +187,7 @@ def unpackb(bytes packed_bytes, object object_hook=None, object list_hook=None): if not PyCallable_Check(list_hook): raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook - ret = template_execute(&ctx, p, len(packed_bytes), &off) + ret = template_execute(&ctx, buf, buf_len, &off) if ret == 1: return template_data(&ctx) else: @@ -216,8 +197,8 @@ unpacks = unpackb def unpack(object stream, object object_hook=None, object list_hook=None): """unpack an object from stream.""" - packed = stream.read() - return unpackb(packed, object_hook=object_hook, list_hook=list_hook) + return unpackb(stream.read(), + object_hook=object_hook, list_hook=list_hook) cdef class UnpackIterator(object): cdef object unpacker @@ -232,21 +213,12 @@ cdef class UnpackIterator(object): return self cdef class Unpacker(object): - """Unpacker(file_like=None, read_size=1024*1024) + """Unpacker(read_size=1024*1024) Streaming unpacker. - file_like must have read(n) method. read_size is used like file_like.read(read_size) - If file_like is None, you can ``feed()`` bytes. ``feed()`` is - useful for unpacking from non-blocking stream. - - exsample 1: - unpacker = Unpacker(afile) - for o in unpacker: - do_something(o) - - example 2: + example: unpacker = Unpacker() while 1: buf = astream.read() @@ -254,13 +226,11 @@ cdef class Unpacker(object): for o in unpacker: do_something(o) """ - cdef template_context ctx cdef char* buf cdef size_t buf_size, buf_head, buf_tail cdef object file_like cdef int read_size - cdef object waiting_bytes cdef bint use_list cdef object object_hook @@ -268,8 +238,7 @@ cdef class Unpacker(object): self.buf = NULL def __dealloc__(self): - if self.buf: - free(self.buf); + free(self.buf); def __init__(self, file_like=None, int read_size=0, bint use_list=0, object object_hook=None, object list_hook=None): @@ -278,7 +247,6 @@ cdef class Unpacker(object): self.use_list = use_list self.file_like = file_like self.read_size = read_size - self.waiting_bytes = [] self.buf = malloc(read_size) self.buf_size = read_size self.buf_head = 0 @@ -295,65 +263,49 @@ cdef class Unpacker(object): raise TypeError("object_hook must be a callable.") self.ctx.user.list_hook = list_hook - def feed(self, bytes next_bytes): - self.waiting_bytes.append(next_bytes) + def feed(self, object next_bytes): + cdef char* buf + cdef Py_ssize_t buf_len + PyObject_AsReadBuffer(next_bytes, &buf, &buf_len) + self.append_buffer(buf, buf_len) - cdef append_buffer(self): - cdef char* buf = self.buf - cdef Py_ssize_t tail = self.buf_tail - cdef Py_ssize_t l - cdef bytes b + cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len): + cdef: + char* buf = self.buf + size_t head = self.buf_head + size_t tail = self.buf_tail + size_t buf_size = self.buf_size + size_t new_size - for b in self.waiting_bytes: - l = len(b) - memcpy(buf + tail, (b), l) - tail += l - self.buf_tail = tail - del self.waiting_bytes[:] + if tail + _buf_len > buf_size: + if ((tail - head) + _buf_len)*2 < buf_size: + # move to front. + memmove(buf, buf + head, tail - head) + tail -= head + head = 0 + else: + # expand buffer. + new_size = tail + _buf_len + if new_size < buf_size*2: + new_size = buf_size*2 + buf = realloc(buf, new_size) + buf_size = new_size - # prepare self.buf + memcpy(buf + tail, (_buf), _buf_len) + self.buf_head = head + self.buf_size = buf_size + self.buf_tail = tail + _buf_len + + # prepare self.buf from file_like cdef fill_buffer(self): - cdef Py_ssize_t add_size - if self.file_like is not None: next_bytes = self.file_like.read(self.read_size) if next_bytes: - self.waiting_bytes.append(next_bytes) + self.append_buffer(PyBytes_AsString(next_bytes), + PyBytes_Size(next_bytes)) else: self.file_like = None - if not self.waiting_bytes: - return - - add_size = 0 - for b in self.waiting_bytes: - add_size += len(b) - - cdef char* buf = self.buf - cdef size_t head = self.buf_head - cdef size_t tail = self.buf_tail - cdef size_t size = self.buf_size - - if self.buf_tail + add_size <= self.buf_size: - # do nothing. - pass - if self.buf_tail - self.buf_head + add_size < self.buf_size: - # move to front. - memmove(buf, buf + head, tail - head) - tail -= head - head = 0 - else: - # expand buffer - size = tail + add_size - buf = realloc(buf, size) - - self.buf = buf - self.buf_head = head - self.buf_tail = tail - self.buf_size = size - - self.append_buffer() - cpdef unpack(self): """unpack one object""" cdef int ret diff --git a/test/test_buffer.py b/test/test_buffer.py index 436b554..ce7a72d 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -7,10 +7,10 @@ from msgpack import packb, unpackb def test_unpack_buffer(): from array import array - buf = array('b') - buf.fromstring(packb(['foo', 'bar'])) + buf = array('c') + buf.fromstring(packb(('foo', 'bar'))) obj = unpackb(buf) - assert_equal(['foo', 'bar'], obj) + assert_equal(('foo', 'bar'), obj) if __name__ == '__main__': main() diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 789ccd2..df6e308 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,22 +1,22 @@ #!/usr/bin/env python # coding: utf-8 -from __future__ import unicode_literals, print_function +from __future__ import unicode_literals from msgpack import Unpacker def test_foobar(): unpacker = Unpacker(read_size=3) unpacker.feed(b'foobar') - assert unpacker.unpack() == ord('f') - assert unpacker.unpack() == ord('o') - assert unpacker.unpack() == ord('o') - assert unpacker.unpack() == ord('b') - assert unpacker.unpack() == ord('a') - assert unpacker.unpack() == ord('r') + assert unpacker.unpack() == ord(b'f') + assert unpacker.unpack() == ord(b'o') + assert unpacker.unpack() == ord(b'o') + assert unpacker.unpack() == ord(b'b') + assert unpacker.unpack() == ord(b'a') + assert unpacker.unpack() == ord(b'r') try: o = unpacker.unpack() - print("Oops!", o) + print "Oops!", o assert 0 except StopIteration: assert 1 From 2c9fddf085fee3fe16f86f979e42ea87cfaaa084 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 3 Nov 2010 03:15:12 +0900 Subject: [PATCH 0468/1172] python: Port some tests from 2 to 3. --- test3/test_buffer.py | 16 ++++++++++++++++ test3/test_obj.py | 13 +++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 test3/test_buffer.py diff --git a/test3/test_buffer.py b/test3/test_buffer.py new file mode 100644 index 0000000..01310a0 --- /dev/null +++ b/test3/test_buffer.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * +from msgpack import packb, unpackb + +def test_unpack_buffer(): + from array import array + buf = array('b') + buf.fromstring(packb(('foo', 'bar'))) + obj = unpackb(buf) + assert_equal((b'foo', b'bar'), obj) + +if __name__ == '__main__': + main() diff --git a/test3/test_obj.py b/test3/test_obj.py index 972d1ca..236988d 100644 --- a/test3/test_obj.py +++ b/test3/test_obj.py @@ -26,6 +26,19 @@ def test_decode_hook(): unpacked = unpacks(packed, object_hook=_decode_complex) eq_(unpacked[1], 1+2j) +@raises(TypeError) +def test_bad_hook(): + packed = packs([3, 1+2j], default=lambda o: o) + unpacked = unpacks(packed) + +def _arr_to_str(arr): + return ''.join(str(c) for c in arr) + +def test_array_hook(): + packed = packs([1,2,3]) + unpacked = unpacks(packed, list_hook=_arr_to_str) + eq_(unpacked, '123') + if __name__ == '__main__': #main() test_decode_hook() From 3124576e1e177831632b84792b6297e7a1078ea0 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 3 Nov 2010 03:26:33 +0900 Subject: [PATCH 0469/1172] python: Add ChangeLog.rst --- ChangeLog.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 ChangeLog.rst diff --git a/ChangeLog.rst b/ChangeLog.rst new file mode 100644 index 0000000..022f955 --- /dev/null +++ b/ChangeLog.rst @@ -0,0 +1,16 @@ +0.2.0 +====== +:release date: NOT RELEASED YET + +New feature +------------ +* Add *object_hook* and *list_hook* option to unpacker. It allows you to + hook unpacing mapping type and array type. + +* Add *default* option to packer. It allows you to pack unsupported types. + +* unpacker accepts (old) buffer types. + +Bugs fixed +---------- +* Compilation error on win32. From 948f4d00c6e50e90fc99434905e8a1ce3f708411 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 3 Nov 2010 03:35:52 +0900 Subject: [PATCH 0470/1172] python: Add ws2_32 library if platform is win32. --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ac7ece5..aedd990 100755 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ version = (0, 1, 6, 'final') import os +import sys from glob import glob from distutils.core import setup, Extension from distutils.command.sdist import sdist @@ -42,10 +43,13 @@ else: Sdist = sdist +libraries = ['ws2_32'] if sys.platform == 'win32' else [] + msgpack_mod = Extension('msgpack._msgpack', sources=sources, + libraries=libraries, ) -del sources +del sources, libraries desc = 'MessagePack (de)serializer.' @@ -69,6 +73,7 @@ setup(name='msgpack-python', url='http://msgpack.sourceforge.net/', download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ + 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', From 9d04822f35fde2eb46136b57bf73a23a6d3954f6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 3 Nov 2010 03:43:49 +0900 Subject: [PATCH 0471/1172] python: 0.1.7 --- ChangeLog.rst | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 022f955..77e1707 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,6 @@ -0.2.0 +0.1.7 ====== -:release date: NOT RELEASED YET +:release date: 2010-11-02 New feature ------------ diff --git a/setup.py b/setup.py index aedd990..d21e6b5 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 1, 6, 'final') +version = (0, 1, 7, 'final') import os import sys From 67b80f8b8a252e42119b3e9dce151a3354a58872 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 9 Jan 2011 23:06:56 +0900 Subject: [PATCH 0472/1172] python: Update ChangeLog. --- ChangeLog.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 77e1707..19cc6b6 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,23 @@ +0.1.8 +====== +:release date: NOT YET RELEASED + +New feature +------------ +* Support ``loads`` and ``dumps`` aliases for API compatibility with + simplejson and pickle. + +* Add *object_hook* and *list_hook* option to unpacker. It allows you to + hook unpacing mapping type and array type. + +* Add *default* option to packer. It allows you to pack unsupported types. + +* unpacker accepts (old) buffer types. + +Bugs fixed +---------- + + 0.1.7 ====== :release date: 2010-11-02 From 9c61ce2c6cb8f87d6d928ef728a882bf141ccaf2 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 9 Jan 2011 23:13:35 +0900 Subject: [PATCH 0473/1172] python: Don't use ``from __future__ import unicode_literals``. Python 2.5 or older doesn't support it. --- test/test_sequnpack.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index df6e308..c92658c 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,13 +1,11 @@ #!/usr/bin/env python # coding: utf-8 -from __future__ import unicode_literals - from msgpack import Unpacker def test_foobar(): unpacker = Unpacker(read_size=3) - unpacker.feed(b'foobar') + unpacker.feed('foobar') assert unpacker.unpack() == ord(b'f') assert unpacker.unpack() == ord(b'o') assert unpacker.unpack() == ord(b'o') From 569729c3c218cdbb87b486524ab1e71c1f8bba61 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 9 Jan 2011 23:17:20 +0900 Subject: [PATCH 0474/1172] python: Make aliases for API compatibility to pickle. ``dumps`` is alias of ``packb`` and ``loads`` is alias of ``unpacks``. --- msgpack/_msgpack.pyx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 5394055..3316a22 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -73,7 +73,6 @@ cdef class Packer(object): if o is None: ret = msgpack_pack_nil(&self.pk) - #elif PyBool_Check(o): elif isinstance(o, bool): if o: ret = msgpack_pack_true(&self.pk) @@ -145,7 +144,7 @@ def packb(object o, default=None): packer = Packer(default=default) return packer.pack(o) -packs = packb +dumps = packs = packb cdef extern from "unpack.h": ctypedef struct msgpack_user: @@ -193,7 +192,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None): else: return None -unpacks = unpackb +loads = unpacks = unpackb def unpack(object stream, object object_hook=None, object list_hook=None): """unpack an object from stream.""" From 5dbf2f5ef73c574d6d2d51b32abfa63b3b124ae1 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 9 Jan 2011 23:29:18 +0900 Subject: [PATCH 0475/1172] python: Add test for issue29. --- test/test_seq.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 test/test_seq.py diff --git a/test/test_seq.py b/test/test_seq.py new file mode 100644 index 0000000..993a59e --- /dev/null +++ b/test/test_seq.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * + +import StringIO +import msgpack + +binarydata = [chr(i) for i in xrange(256)] +binarydata = "".join(binarydata) + +def gen_binary_data(idx): + data = binarydata[:idx % 300] + return data + +def test_exceeding_unpacker_read_size(): + dumpf = StringIO.StringIO() + + packer = msgpack.Packer() + + NUMBER_OF_STRINGS = 6 + read_size = 16 + # 5 ok for read_size=16, while 6 glibc detected *** python: double free or corruption (fasttop): + # 20 ok for read_size=256, while 25 segfaults / glibc detected *** python: double free or corruption (!prev) + # 40 ok for read_size=1024, while 50 introduces errors + # 7000 ok for read_size=1024*1024, while 8000 leads to glibc detected *** python: double free or corruption (!prev): + + for idx in xrange(NUMBER_OF_STRINGS): + data = gen_binary_data(idx) + dumpf.write(packer.pack(data)) + + f = StringIO.StringIO(dumpf.getvalue()) + dumpf.close() + + unpacker = msgpack.Unpacker(f, read_size=read_size) + + read_count = 0 + for idx, o in enumerate(unpacker): + assert_equal(type(o), str) + assert_equal(o, gen_binary_data(idx)) + read_count += 1 + + assert_equal(read_count, NUMBER_OF_STRINGS) + + +if __name__ == '__main__': + # main() + test_exceeding_unpacker_read_size() From ed3f3899c1e22fc5dc61e88af3feee9029df6ec4 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 9 Jan 2011 23:40:09 +0900 Subject: [PATCH 0476/1172] python: Fix segv on unpacking from stream. --- msgpack/_msgpack.pyx | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 3316a22..5afa961 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -229,7 +229,8 @@ cdef class Unpacker(object): cdef char* buf cdef size_t buf_size, buf_head, buf_tail cdef object file_like - cdef int read_size + cdef object file_like_read + cdef Py_ssize_t read_size cdef bint use_list cdef object object_hook @@ -239,12 +240,16 @@ cdef class Unpacker(object): def __dealloc__(self): free(self.buf); - def __init__(self, file_like=None, int read_size=0, bint use_list=0, + def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, object object_hook=None, object list_hook=None): if read_size == 0: read_size = 1024*1024 self.use_list = use_list self.file_like = file_like + if file_like: + self.file_like_read = file_like.read + if not PyCallable_Check(self.file_like_read): + raise ValueError("`file_like.read` must be a callable.") self.read_size = read_size self.buf = malloc(read_size) self.buf_size = read_size @@ -265,6 +270,9 @@ cdef class Unpacker(object): def feed(self, object next_bytes): cdef char* buf cdef Py_ssize_t buf_len + if self.file_like is not None: + raise AssertionError( + "unpacker.feed() is not be able to use with`file_like`.") PyObject_AsReadBuffer(next_bytes, &buf, &buf_len) self.append_buffer(buf, buf_len) @@ -298,7 +306,7 @@ cdef class Unpacker(object): # prepare self.buf from file_like cdef fill_buffer(self): if self.file_like is not None: - next_bytes = self.file_like.read(self.read_size) + next_bytes = self.file_like_read(self.read_size) if next_bytes: self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes)) @@ -308,18 +316,19 @@ cdef class Unpacker(object): cpdef unpack(self): """unpack one object""" cdef int ret - self.fill_buffer() - ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) - if ret == 1: - o = template_data(&self.ctx) - template_init(&self.ctx) - return o - elif ret == 0: - if self.file_like is not None: - return self.unpack() - raise StopIteration, "No more unpack data." - else: - raise ValueError, "Unpack failed." + while 1: + ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + if ret == 1: + o = template_data(&self.ctx) + template_init(&self.ctx) + return o + elif ret == 0: + if self.file_like is not None: + self.fill_buffer() + continue + raise StopIteration("No more unpack data.") + else: + raise ValueError("Unpack failed: error = %d" % (ret,)) def __iter__(self): return UnpackIterator(self) From 74e7ebea4f5f93dda9b266eae8019635b6142eef Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 9 Jan 2011 23:54:06 +0900 Subject: [PATCH 0477/1172] python: Fix another segv. --- msgpack/_msgpack.pyx | 1 + 1 file changed, 1 insertion(+) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 5afa961..0dd926f 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -299,6 +299,7 @@ cdef class Unpacker(object): buf_size = new_size memcpy(buf + tail, (_buf), _buf_len) + self.buf = buf self.buf_head = head self.buf_size = buf_size self.buf_tail = tail + _buf_len From 013d1da68f7ddc27f9a748276f3edb57b1173e02 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 9 Jan 2011 23:58:26 +0900 Subject: [PATCH 0478/1172] python: msgpack-python-0.1.8 --- ChangeLog.rst | 3 ++- setup.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 19cc6b6..4c70ad5 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,6 @@ 0.1.8 ====== -:release date: NOT YET RELEASED +:release date: 2011-01-10 New feature ------------ @@ -16,6 +16,7 @@ New feature Bugs fixed ---------- +* Fix segv around ``Unpacker.feed`` or ``Unpacker(file)``. 0.1.7 diff --git a/setup.py b/setup.py index d21e6b5..216d255 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 1, 7, 'final') +version = (0, 1, 8, 'final') import os import sys @@ -20,6 +20,8 @@ except ImportError: f = open('msgpack/__version__.py', 'w') f.write("version = %r\n" % (version,)) f.close() +del f + version_str = '.'.join(str(x) for x in version[:3]) if len(version) > 3 and version[3] != 'final': version_str += version[3] From 8e38a074bbc5d3546c0eb049b838aa3afa3c5f3a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 10 Jan 2011 00:10:29 +0900 Subject: [PATCH 0479/1172] python: Change URL in setup.py from sf.net to msgpack.org --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 216d255..ddace35 100755 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ del sources, libraries desc = 'MessagePack (de)serializer.' long_desc = """MessagePack (de)serializer for Python. -What's MessagePack? (from http://msgpack.sourceforge.net/) +What's MessagePack? (from http://msgpack.org/) MessagePack is a binary-based efficient data interchange format that is focused on high performance. It is like JSON, but very fast and small. @@ -72,7 +72,7 @@ setup(name='msgpack-python', packages=['msgpack'], description=desc, long_description=long_desc, - url='http://msgpack.sourceforge.net/', + url='http://msgpack.org/', download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ 'Programming Language :: Python :: 2', From 56aaed36416ebe8f7df670d21b04219e67e6e2c9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 10 Jan 2011 05:05:14 +0900 Subject: [PATCH 0480/1172] python: Fix typo in docstring. (thanks to Mateusz.) --- msgpack/_msgpack.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 0dd926f..c7eec81 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -264,7 +264,7 @@ cdef class Unpacker(object): self.ctx.user.object_hook = object_hook if list_hook is not None: if not PyCallable_Check(list_hook): - raise TypeError("object_hook must be a callable.") + raise TypeError("list_hook must be a callable.") self.ctx.user.list_hook = list_hook def feed(self, object next_bytes): From 47b0c273bb70d34b253c9e908a3f2d55e99b8cef Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 10 Jan 2011 05:07:07 +0900 Subject: [PATCH 0481/1172] python: Check if (m|re)alloc's return value is NULL. (Thanks to Mateusz) --- msgpack/_msgpack.pyx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index c7eec81..e48f8b2 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -239,6 +239,7 @@ cdef class Unpacker(object): def __dealloc__(self): free(self.buf); + self.buf = NULL; def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, object object_hook=None, object list_hook=None): @@ -252,6 +253,8 @@ cdef class Unpacker(object): raise ValueError("`file_like.read` must be a callable.") self.read_size = read_size self.buf = malloc(read_size) + if self.buf == NULL: + raise MemoryError("Unable to allocate internal buffer.") self.buf_size = read_size self.buf_head = 0 self.buf_tail = 0 @@ -295,7 +298,9 @@ cdef class Unpacker(object): new_size = tail + _buf_len if new_size < buf_size*2: new_size = buf_size*2 - buf = realloc(buf, new_size) + buf = realloc(buf, new_size) + if buf == NULL: + raise MemoryError("Unable to enlarge internal buffer.") # self.buf still holds old buffer and will be freed during obj destruction buf_size = new_size memcpy(buf + tail, (_buf), _buf_len) From b453385d92e549204e7e1639f286589a38143061 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 10 Jan 2011 20:47:23 +0900 Subject: [PATCH 0482/1172] python: Add memory error check. --- msgpack/_msgpack.pyx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index e48f8b2..9f817dd 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -46,6 +46,8 @@ cdef class Packer(object): def __cinit__(self): cdef int buf_size = 1024*1024 self.pk.buf = malloc(buf_size); + if self.pk.buf == NULL: + raise MemoryError("Unable to allocate internal buffer.") self.pk.buf_size = buf_size self.pk.length = 0 @@ -300,7 +302,9 @@ cdef class Unpacker(object): new_size = buf_size*2 buf = realloc(buf, new_size) if buf == NULL: - raise MemoryError("Unable to enlarge internal buffer.") # self.buf still holds old buffer and will be freed during obj destruction + # self.buf still holds old buffer and will be freed during + # obj destruction + raise MemoryError("Unable to enlarge internal buffer.") buf_size = new_size memcpy(buf + tail, (_buf), _buf_len) From 77a97b9c16bfabe1f8816c348d65e1d8277780d6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 28 Jan 2011 18:59:05 +0900 Subject: [PATCH 0483/1172] Add use_list option to unpack and unpackb --- msgpack/_msgpack.pyx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 9f817dd..9564171 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -167,7 +167,7 @@ cdef extern from "unpack.h": object template_data(template_context* ctx) -def unpackb(object packed, object object_hook=None, object list_hook=None): +def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0): """Unpack packed_bytes to object. Returns an unpacked object.""" cdef template_context ctx cdef size_t off = 0 @@ -178,7 +178,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None): PyObject_AsReadBuffer(packed, &buf, &buf_len) template_init(&ctx) - ctx.user.use_list = 0 + ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL if object_hook is not None: if not PyCallable_Check(object_hook): @@ -196,9 +196,9 @@ def unpackb(object packed, object object_hook=None, object list_hook=None): loads = unpacks = unpackb -def unpack(object stream, object object_hook=None, object list_hook=None): +def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0): """unpack an object from stream.""" - return unpackb(stream.read(), + return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, list_hook=list_hook) cdef class UnpackIterator(object): From d0de96cacd3bd5912994c7b671a5fbf6cb303d31 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 29 Jan 2011 07:27:10 +0900 Subject: [PATCH 0484/1172] python: refactoring. --- msgpack/_msgpack.pyx | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 9564171..4df0345 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -41,7 +41,7 @@ cdef class Packer(object): astream.write(packer.pack(b)) """ cdef msgpack_packer pk - cdef object default + cdef object _default def __cinit__(self): cdef int buf_size = 1024*1024 @@ -55,13 +55,12 @@ cdef class Packer(object): if default is not None: if not PyCallable_Check(default): raise TypeError("default must be a callable.") - self.default = default + self._default = default def __dealloc__(self): free(self.pk.buf); - cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT, - default=None) except -1: + cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: cdef long long llval cdef unsigned long long ullval cdef long longval @@ -109,18 +108,18 @@ cdef class Packer(object): ret = msgpack_pack_map(&self.pk, len(d)) if ret == 0: for k,v in d.items(): - ret = self._pack(k, nest_limit-1, default) + ret = self._pack(k, nest_limit-1) if ret != 0: break - ret = self._pack(v, nest_limit-1, default) + ret = self._pack(v, nest_limit-1) if ret != 0: break elif PySequence_Check(o): ret = msgpack_pack_array(&self.pk, len(o)) if ret == 0: for v in o: - ret = self._pack(v, nest_limit-1, default) + ret = self._pack(v, nest_limit-1) if ret != 0: break - elif default is not None: - o = self.default(o) + elif self._default is not None: + o = self._default(o) ret = self._pack(o, nest_limit) else: raise TypeError("can't serialize %r" % (o,)) @@ -128,7 +127,7 @@ cdef class Packer(object): def pack(self, object obj): cdef int ret - ret = self._pack(obj, DEFAULT_RECURSE_LIMIT, self.default) + ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) if ret: raise TypeError buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) From 9e8261cbd94ca146bdf2fa72c73a19468f96ca74 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 29 Jan 2011 07:31:06 +0900 Subject: [PATCH 0485/1172] python: 0.1.9 --- ChangeLog.rst | 14 ++++++++++++++ setup.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 4c70ad5..75e86b2 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,17 @@ +0.1.9 +====== +:release date: 2011-01-29 + +New feature +----------- +* ``use_list`` option is added to unpack(b) like Unpacker. + (Use keyword argument because order of parameters are different) + +Bugs fixed +---------- +* Fix typo. +* Add MemoryError check. + 0.1.8 ====== :release date: 2011-01-10 diff --git a/setup.py b/setup.py index ddace35..16987a4 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 1, 8, 'final') +version = (0, 1, 9, 'final') import os import sys From 3aaf5f5a7afb4d51e6384f87a3f70a4235b9afb5 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 29 Jan 2011 23:22:41 +0900 Subject: [PATCH 0486/1172] python: Fix segmentation fault when `default` returns it's argument. --- msgpack/_msgpack.pyx | 4 ++-- test/test_obj.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 4df0345..9a6c232 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -118,9 +118,9 @@ cdef class Packer(object): for v in o: ret = self._pack(v, nest_limit-1) if ret != 0: break - elif self._default is not None: + elif self._default: o = self._default(o) - ret = self._pack(o, nest_limit) + ret = self._pack(o, nest_limit-1) else: raise TypeError("can't serialize %r" % (o,)) return ret diff --git a/test/test_obj.py b/test/test_obj.py index bc85736..0eebe7b 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -26,7 +26,7 @@ def test_decode_hook(): unpacked = unpacks(packed, object_hook=_decode_complex) eq_(unpacked[1], 1+2j) -@raises(TypeError) +@raises(ValueError) def test_bad_hook(): packed = packs([3, 1+2j], default=lambda o: o) unpacked = unpacks(packed) From 60d3ce3a180789ef5f57c22dd579c383ea7eed91 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 29 Jan 2011 23:23:56 +0900 Subject: [PATCH 0487/1172] python: Disable gc while deserializing. --- msgpack/_msgpack.pyx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 9a6c232..e367444 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -9,6 +9,9 @@ cdef extern from "Python.h": from libc.stdlib cimport * from libc.string cimport * +import gc +_gc_disable = gc.disable +_gc_enable = gc.enable cdef extern from "pack.h": struct msgpack_packer: @@ -187,7 +190,9 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint if not PyCallable_Check(list_hook): raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook + _gc_disable() ret = template_execute(&ctx, buf, buf_len, &off) + _gc_enable() if ret == 1: return template_data(&ctx) else: @@ -326,7 +331,9 @@ cdef class Unpacker(object): """unpack one object""" cdef int ret while 1: + _gc_disable() ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + _gc_enable() if ret == 1: o = template_data(&self.ctx) template_init(&self.ctx) From af7113bb31877f337eef3d43048c0a9f1cb74258 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 30 Jan 2011 10:45:39 +0900 Subject: [PATCH 0488/1172] python: Remove UnpackIterator. Unpacker is iterator of itself. --- msgpack/_msgpack.pyx | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index e367444..cdcd0c8 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -205,18 +205,6 @@ def unpack(object stream, object object_hook=None, object list_hook=None, bint u return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, list_hook=list_hook) -cdef class UnpackIterator(object): - cdef object unpacker - - def __init__(self, unpacker): - self.unpacker = unpacker - - def __next__(self): - return self.unpacker.unpack() - - def __iter__(self): - return self - cdef class Unpacker(object): """Unpacker(read_size=1024*1024) @@ -347,7 +335,10 @@ cdef class Unpacker(object): raise ValueError("Unpack failed: error = %d" % (ret,)) def __iter__(self): - return UnpackIterator(self) + return self + + def __next__(self): + return self.unpack() # for debug. #def _buf(self): From 752e3d1b783fc1c12a28e05a93aa73ac7c6b751c Mon Sep 17 00:00:00 2001 From: tailhook Date: Fri, 15 Apr 2011 17:36:17 +0300 Subject: [PATCH 0489/1172] Implemented encoding for strings * Packer by default uses `utf-8` encoding by default * Unpacker uses `None` by default, so no decoding is done * Both pack and unpack has `encoding` and `unicode_errors` arguments, if `encoding` is `None` no encoding/decoding is done, otherwise it is python codec. `unicode_errors` is supplied as `errors` parameter to codec --- msgpack/_msgpack.pyx | 72 ++++++++++++++++++++++++++++++++++---------- msgpack/unpack.h | 8 ++++- test/test_pack.py | 57 ++++++++++++++++++++++++++++++++--- test3/test_obj.py | 2 +- test3/test_pack.py | 55 +++++++++++++++++++++++++++++++-- 5 files changed, 169 insertions(+), 25 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index cdcd0c8..443cbd7 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -36,7 +36,7 @@ cdef int DEFAULT_RECURSE_LIMIT=511 cdef class Packer(object): """MessagePack Packer - + usage: packer = Packer() @@ -45,6 +45,10 @@ cdef class Packer(object): """ cdef msgpack_packer pk cdef object _default + cdef object _bencoding + cdef object _berrors + cdef char *encoding + cdef char *unicode_errors def __cinit__(self): cdef int buf_size = 1024*1024 @@ -54,11 +58,25 @@ cdef class Packer(object): self.pk.buf_size = buf_size self.pk.length = 0 - def __init__(self, default=None): + def __init__(self, default=None, encoding='utf-8', unicode_errors='strict'): if default is not None: if not PyCallable_Check(default): raise TypeError("default must be a callable.") self._default = default + if encoding is None: + self.encoding = NULL + self.unicode_errors = NULL + else: + if isinstance(encoding, unicode): + self._bencoding = encoding.encode('ascii') + else: + self._bencoding = encoding + self.encoding = PyBytes_AsString(self._bencoding) + if isinstance(unicode_errors, unicode): + self._berrors = unicode_errors.encode('ascii') + else: + self._berrors = unicode_errors + self.unicode_errors = PyBytes_AsString(self._berrors) def __dealloc__(self): free(self.pk.buf); @@ -68,7 +86,7 @@ cdef class Packer(object): cdef unsigned long long ullval cdef long longval cdef double fval - cdef char* rawval + cdef char* rawval cdef int ret cdef dict d @@ -101,7 +119,9 @@ cdef class Packer(object): if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyUnicode_Check(o): - o = PyUnicode_AsUTF8String(o) + if not self.encoding: + raise TypeError("Can't encode utf-8 no encoding is specified") + o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) rawval = o ret = msgpack_pack_raw(&self.pk, len(o)) if ret == 0: @@ -138,14 +158,14 @@ cdef class Packer(object): return buf -def pack(object o, object stream, default=None): +def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'): """pack an object `o` and write it to stream).""" - packer = Packer(default=default) + packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors) stream.write(packer.pack(o)) -def packb(object o, default=None): +def packb(object o, default=None, encoding='utf-8', unicode_errors='strict'): """pack o and return packed bytes.""" - packer = Packer(default=default) + packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors) return packer.pack(o) dumps = packs = packb @@ -155,6 +175,8 @@ cdef extern from "unpack.h": int use_list PyObject* object_hook PyObject* list_hook + char *encoding + char *unicode_errors ctypedef struct template_context: msgpack_user user @@ -164,12 +186,12 @@ cdef extern from "unpack.h": PyObject* key int template_execute(template_context* ctx, const_char_ptr data, - size_t len, size_t* off) + size_t len, size_t* off) except -1 void template_init(template_context* ctx) object template_data(template_context* ctx) -def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0): +def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"): """Unpack packed_bytes to object. Returns an unpacked object.""" cdef template_context ctx cdef size_t off = 0 @@ -179,9 +201,25 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint cdef Py_ssize_t buf_len PyObject_AsReadBuffer(packed, &buf, &buf_len) + if encoding is None: + enc = NULL + else: + if isinstance(encoding, unicode): + bencoding = encoding.encode('ascii') + else: + bencoding = encoding + if isinstance(unicode_errors, unicode): + berrors = unicode_errors.encode('ascii') + else: + berrors = unicode_errors + enc = PyBytes_AsString(bencoding) + err = PyBytes_AsString(berrors) + template_init(&ctx) ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL + ctx.user.encoding = enc + ctx.user.unicode_errors = err if object_hook is not None: if not PyCallable_Check(object_hook): raise TypeError("object_hook must be a callable.") @@ -191,8 +229,10 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook _gc_disable() - ret = template_execute(&ctx, buf, buf_len, &off) - _gc_enable() + try: + ret = template_execute(&ctx, buf, buf_len, &off) + finally: + _gc_enable() if ret == 1: return template_data(&ctx) else: @@ -200,10 +240,10 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint loads = unpacks = unpackb -def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0): +def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"): """unpack an object from stream.""" return unpackb(stream.read(), use_list=use_list, - object_hook=object_hook, list_hook=list_hook) + object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors) cdef class Unpacker(object): """Unpacker(read_size=1024*1024) @@ -236,7 +276,7 @@ cdef class Unpacker(object): self.buf = NULL; def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, - object object_hook=None, object list_hook=None): + object object_hook=None, object list_hook=None, encoding=None, unicode_errors=None): if read_size == 0: read_size = 1024*1024 self.use_list = use_list @@ -292,7 +332,7 @@ cdef class Unpacker(object): new_size = tail + _buf_len if new_size < buf_size*2: new_size = buf_size*2 - buf = realloc(buf, new_size) + buf = realloc(buf, new_size) if buf == NULL: # self.buf still holds old buffer and will be freed during # obj destruction diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 453ec2b..0586ca8 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -23,6 +23,8 @@ typedef struct unpack_user { int use_list; PyObject *object_hook; PyObject *list_hook; + const char *encoding; + const char *unicode_errors; } unpack_user; @@ -197,7 +199,11 @@ static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_objec static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { PyObject *py; - py = PyBytes_FromStringAndSize(p, l); + if(u->encoding) { + py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors); + } else { + py = PyBytes_FromStringAndSize(p, l); + } if (!py) return -1; *o = py; diff --git a/test/test_pack.py b/test/test_pack.py index 5dec068..2aef588 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -15,14 +15,63 @@ def testPack(): 0, 1, 127, 128, 255, 256, 65535, 65536, -1, -32, -33, -128, -129, -32768, -32769, 1.0, - "", "a", "a"*31, "a"*32, + b"", b"a", b"a"*31, b"a"*32, None, True, False, - (), ((),), ((), None,), - {None: 0}, - (1<<23), + (), ((),), ((), None,), + {None: 0}, + (1<<23), ] for td in test_data: check(td) +def testPackUnicode(): + test_data = [ + u"", u"abcd", (u"defgh",), u"РуÑÑкий текÑÑ‚", + ] + for td in test_data: + re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8') + assert_equal(re, td) + +def testPackUTF32(): + test_data = [ + u"", u"abcd", (u"defgh",), u"РуÑÑкий текÑÑ‚", + ] + for td in test_data: + print(packs(td, encoding='utf-32')) + re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32') + assert_equal(re, td) + +def testPackBytes(): + test_data = [ + b"", b"abcd", (b"defgh",), + ] + for td in test_data: + check(td) + +def testIgnoreUnicodeErrors(): + re = unpacks(packs(b'abc\xeddef'), + encoding='utf-8', unicode_errors='ignore') + assert_equal(re, "abcdef") + +@raises(UnicodeDecodeError) +def testStrictUnicodeUnpack(): + unpacks(packs(b'abc\xeddef'), encoding='utf-8') + +@raises(UnicodeEncodeError) +def testStrictUnicodePack(): + packs(u"abc\xeddef", encoding='ascii', unicode_errors='strict') + +def testIgnoreErrorsPack(): + re = unpacks(packs(u"abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8') + assert_equal(re, u"abcdef") + +@raises(TypeError) +def testNoEncoding(): + packs(u"abc", encoding=None) + +def testDecodeBinary(): + re = unpacks(packs(u"abc"), encoding=None) + assert_equal(re, b"abc") + if __name__ == '__main__': main() diff --git a/test3/test_obj.py b/test3/test_obj.py index 236988d..b54021f 100644 --- a/test3/test_obj.py +++ b/test3/test_obj.py @@ -26,7 +26,7 @@ def test_decode_hook(): unpacked = unpacks(packed, object_hook=_decode_complex) eq_(unpacked[1], 1+2j) -@raises(TypeError) +@raises(ValueError) def test_bad_hook(): packed = packs([3, 1+2j], default=lambda o: o) unpacked = unpacks(packed) diff --git a/test3/test_pack.py b/test3/test_pack.py index c861704..e53f7e6 100644 --- a/test3/test_pack.py +++ b/test3/test_pack.py @@ -17,12 +17,61 @@ def testPack(): 1.0, b"", b"a", b"a"*31, b"a"*32, None, True, False, - (), ((),), ((), None,), - {None: 0}, - (1<<23), + (), ((),), ((), None,), + {None: 0}, + (1<<23), ] for td in test_data: check(td) +def testPackUnicode(): + test_data = [ + "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚", + ] + for td in test_data: + re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8') + assert_equal(re, td) + +def testPackUTF32(): + test_data = [ + "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚", + ] + for td in test_data: + print(packs(td, encoding='utf-32')) + re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32') + assert_equal(re, td) + +def testPackBytes(): + test_data = [ + b"", b"abcd", (b"defgh",), + ] + for td in test_data: + check(td) + +def testIgnoreUnicodeErrors(): + re = unpacks(packs(b'abc\xeddef'), + encoding='utf-8', unicode_errors='ignore') + assert_equal(re, "abcdef") + +@raises(UnicodeDecodeError) +def testStrictUnicodeUnpack(): + unpacks(packs(b'abc\xeddef'), encoding='utf-8') + +@raises(UnicodeEncodeError) +def testStrictUnicodePack(): + packs("abc\xeddef", encoding='ascii', unicode_errors='strict') + +def testIgnoreErrorsPack(): + re = unpacks(packs("abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8') + assert_equal(re, "abcdef") + +@raises(TypeError) +def testNoEncoding(): + packs("abc", encoding=None) + +def testDecodeBinary(): + re = unpacks(packs("abc"), encoding=None) + assert_equal(re, b"abc") + if __name__ == '__main__': main() From 36b0c2de411701699faadf6d7f355b95044e581c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 31 May 2011 14:10:46 +0900 Subject: [PATCH 0490/1172] (python) Change error message for unicode is passed but no encoding is specified. --- msgpack/_msgpack.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 443cbd7..14bc9d7 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -120,7 +120,7 @@ cdef class Packer(object): ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyUnicode_Check(o): if not self.encoding: - raise TypeError("Can't encode utf-8 no encoding is specified") + raise TypeError("Can't pack unicode object: No encoding is specified") o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) rawval = o ret = msgpack_pack_raw(&self.pk, len(o)) From 3ffc75928bfe5b5cf232de996a01436cb4064f65 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 31 May 2011 15:40:11 +0900 Subject: [PATCH 0491/1172] Revert "(python) Change error message for unicode is passed but no encoding is" This reverts commit bd73742552cf16592662a7ec5ba3608888081131. --- ChangeLog.rst | 10 ++++++++++ msgpack/_msgpack.pyx | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 75e86b2..a0aae25 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,13 @@ +0.1.10 +====== +:release date: NOT RELEASED YET + +New feature +----------- +* Add ``encoding`` and ``unicode_erros`` option to packer and unpacker. + When this option is specified, (un)packs unicode object instead of bytes. + This enables using msgpack as a replacement of json. + 0.1.9 ====== :release date: 2011-01-29 diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 14bc9d7..443cbd7 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -120,7 +120,7 @@ cdef class Packer(object): ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyUnicode_Check(o): if not self.encoding: - raise TypeError("Can't pack unicode object: No encoding is specified") + raise TypeError("Can't encode utf-8 no encoding is specified") o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) rawval = o ret = msgpack_pack_raw(&self.pk, len(o)) From ff594d71dd85661e6307a9cca6ed184227e1dd35 Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 1 Jun 2011 18:30:43 +0900 Subject: [PATCH 0492/1172] (python) make test pass with Python 2.5 --- test/test_pack.py | 33 +++++++++++++++++++-------------- test/test_sequnpack.py | 20 ++++++++++---------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/test/test_pack.py b/test/test_pack.py index 2aef588..2b5f1ad 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -3,6 +3,7 @@ from nose import main from nose.tools import * +from nose.plugins.skip import SkipTest from msgpack import packs, unpacks @@ -15,7 +16,7 @@ def testPack(): 0, 1, 127, 128, 255, 256, 65535, 65536, -1, -32, -33, -128, -129, -32768, -32769, 1.0, - b"", b"a", b"a"*31, b"a"*32, + "", "a", "a"*31, "a"*32, None, True, False, (), ((),), ((), None,), {None: 0}, @@ -33,36 +34,40 @@ def testPackUnicode(): assert_equal(re, td) def testPackUTF32(): - test_data = [ - u"", u"abcd", (u"defgh",), u"РуÑÑкий текÑÑ‚", - ] - for td in test_data: - print(packs(td, encoding='utf-32')) - re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32') - assert_equal(re, td) + try: + test_data = [ + u"", u"abcd", (u"defgh",), u"РуÑÑкий текÑÑ‚", + ] + for td in test_data: + re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32') + assert_equal(re, td) + except LookupError: + raise SkipTest def testPackBytes(): test_data = [ - b"", b"abcd", (b"defgh",), + "", "abcd", ("defgh",), ] for td in test_data: check(td) def testIgnoreUnicodeErrors(): - re = unpacks(packs(b'abc\xeddef'), - encoding='utf-8', unicode_errors='ignore') + re = unpacks(packs('abc\xeddef'), + encoding='ascii', unicode_errors='ignore') assert_equal(re, "abcdef") @raises(UnicodeDecodeError) def testStrictUnicodeUnpack(): - unpacks(packs(b'abc\xeddef'), encoding='utf-8') + unpacks(packs('abc\xeddef'), encoding='utf-8') @raises(UnicodeEncodeError) def testStrictUnicodePack(): packs(u"abc\xeddef", encoding='ascii', unicode_errors='strict') def testIgnoreErrorsPack(): - re = unpacks(packs(u"abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8') + re = unpacks( + packs(u"abcФФФdef", encoding='ascii', unicode_errors='ignore'), + encoding='utf-8') assert_equal(re, u"abcdef") @raises(TypeError) @@ -71,7 +76,7 @@ def testNoEncoding(): def testDecodeBinary(): re = unpacks(packs(u"abc"), encoding=None) - assert_equal(re, b"abc") + assert_equal(re, "abc") if __name__ == '__main__': main() diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index c92658c..d61be23 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -6,12 +6,12 @@ from msgpack import Unpacker def test_foobar(): unpacker = Unpacker(read_size=3) unpacker.feed('foobar') - assert unpacker.unpack() == ord(b'f') - assert unpacker.unpack() == ord(b'o') - assert unpacker.unpack() == ord(b'o') - assert unpacker.unpack() == ord(b'b') - assert unpacker.unpack() == ord(b'a') - assert unpacker.unpack() == ord(b'r') + assert unpacker.unpack() == ord('f') + assert unpacker.unpack() == ord('o') + assert unpacker.unpack() == ord('o') + assert unpacker.unpack() == ord('b') + assert unpacker.unpack() == ord('a') + assert unpacker.unpack() == ord('r') try: o = unpacker.unpack() print "Oops!", o @@ -20,14 +20,14 @@ def test_foobar(): assert 1 else: assert 0 - unpacker.feed(b'foo') - unpacker.feed(b'bar') + unpacker.feed('foo') + unpacker.feed('bar') k = 0 - for o, e in zip(unpacker, b'foobarbaz'): + for o, e in zip(unpacker, 'foobarbaz'): assert o == ord(e) k += 1 - assert k == len(b'foobar') + assert k == len('foobar') if __name__ == '__main__': test_foobar() From cd75ba495af3c153f707c127b0ce6484df6fb1a0 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 24 Jun 2011 00:04:43 +0900 Subject: [PATCH 0493/1172] (python) Fix typo in ChangeLog --- ChangeLog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index a0aae25..10ddf4f 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -4,7 +4,7 @@ New feature ----------- -* Add ``encoding`` and ``unicode_erros`` option to packer and unpacker. +* Add ``encoding`` and ``unicode_errors`` option to packer and unpacker. When this option is specified, (un)packs unicode object instead of bytes. This enables using msgpack as a replacement of json. From 6bc62bd1424b6419e75b99f38199fdfc66d4ec87 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 21 Aug 2011 19:09:36 +0900 Subject: [PATCH 0494/1172] rename README to README.rst --- README => README.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README => README.rst (100%) diff --git a/README b/README.rst similarity index 100% rename from README rename to README.rst From 4a1ce19addc6fc5faf26101b16124401113889d1 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 21 Aug 2011 19:29:33 +0900 Subject: [PATCH 0495/1172] Fix install command for Windows. --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a7e8c53..994492e 100644 --- a/README.rst +++ b/README.rst @@ -28,7 +28,8 @@ Windows MessagePack requires gcc currently. So you need to prepare MinGW GCC. - $ python setup.py install -c mingw32 + $ python setup.py build -c mingw32 + $ python setup.py install TEST ---- From 8c3c8a250b5f6f129e5e077a224ec6916cc87437 Mon Sep 17 00:00:00 2001 From: tailhook Date: Mon, 22 Aug 2011 01:52:45 +0900 Subject: [PATCH 0496/1172] Fixed `encoding` argument for unpacker in Python --- msgpack/_msgpack.pyx | 22 +++++++++++++++++++++- test/test_pack.py | 6 ++++++ test3/test_pack.py | 9 +++++++-- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 443cbd7..5a83ea0 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -203,6 +203,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint if encoding is None: enc = NULL + err = NULL else: if isinstance(encoding, unicode): bencoding = encoding.encode('ascii') @@ -267,6 +268,10 @@ cdef class Unpacker(object): cdef Py_ssize_t read_size cdef bint use_list cdef object object_hook + cdef object _bencoding + cdef object _berrors + cdef char *encoding + cdef char *unicode_errors def __cinit__(self): self.buf = NULL @@ -276,7 +281,8 @@ cdef class Unpacker(object): self.buf = NULL; def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, - object object_hook=None, object list_hook=None, encoding=None, unicode_errors=None): + object object_hook=None, object list_hook=None, + encoding=None, unicode_errors='strict'): if read_size == 0: read_size = 1024*1024 self.use_list = use_list @@ -303,6 +309,20 @@ cdef class Unpacker(object): if not PyCallable_Check(list_hook): raise TypeError("list_hook must be a callable.") self.ctx.user.list_hook = list_hook + if encoding is None: + self.ctx.user.encoding = NULL + self.ctx.user.unicode_errors = NULL + else: + if isinstance(encoding, unicode): + self._bencoding = encoding.encode('ascii') + else: + self._bencoding = encoding + self.ctx.user.encoding = PyBytes_AsString(self._bencoding) + if isinstance(unicode_errors, unicode): + self._berrors = unicode_errors.encode('ascii') + else: + self._berrors = unicode_errors + self.ctx.user.unicode_errors = PyBytes_AsString(self._berrors) def feed(self, object next_bytes): cdef char* buf diff --git a/test/test_pack.py b/test/test_pack.py index 2b5f1ad..357cb3c 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -7,6 +7,8 @@ from nose.plugins.skip import SkipTest from msgpack import packs, unpacks +from StringIO import StringIO + def check(data): re = unpacks(packs(data)) assert_equal(re, data) @@ -32,6 +34,10 @@ def testPackUnicode(): for td in test_data: re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8') assert_equal(re, td) + packer = Packer(encoding='utf-8') + data = packer.pack(td) + re = Unpacker(StringIO(data), encoding='utf-8').unpack() + assert_equal(re, td) def testPackUTF32(): try: diff --git a/test3/test_pack.py b/test3/test_pack.py index e53f7e6..5ff04e7 100644 --- a/test3/test_pack.py +++ b/test3/test_pack.py @@ -4,7 +4,9 @@ from nose import main from nose.tools import * -from msgpack import packs, unpacks +from msgpack import packs, unpacks, Unpacker, Packer + +from io import BytesIO def check(data): re = unpacks(packs(data)) @@ -31,13 +33,16 @@ def testPackUnicode(): for td in test_data: re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8') assert_equal(re, td) + packer = Packer(encoding='utf-8') + data = packer.pack(td) + re = Unpacker(BytesIO(data), encoding='utf-8').unpack() + assert_equal(re, td) def testPackUTF32(): test_data = [ "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚", ] for td in test_data: - print(packs(td, encoding='utf-32')) re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32') assert_equal(re, td) From 60762747b994138dd79c655c8b9da20704f50c0c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 22 Aug 2011 01:57:23 +0900 Subject: [PATCH 0497/1172] Fix error in tests. --- test/test_pack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_pack.py b/test/test_pack.py index 357cb3c..2e1ac22 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -5,7 +5,7 @@ from nose import main from nose.tools import * from nose.plugins.skip import SkipTest -from msgpack import packs, unpacks +from msgpack import packs, unpacks, Packer, Unpacker from StringIO import StringIO From 1a81f6364dcb4356d26b8e3ae48966a9c2a66832 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 22 Aug 2011 02:09:51 +0900 Subject: [PATCH 0498/1172] 0.1.10 --- COPYING | 2 +- ChangeLog.rst | 5 +++-- setup.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/COPYING b/COPYING index 3cb1f1e..f067af3 100644 --- a/COPYING +++ b/COPYING @@ -1,4 +1,4 @@ -Copyright (C) 2008-2010 KLab Inc. +Copyright (C) 2008-2011 INADA Naoki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/ChangeLog.rst b/ChangeLog.rst index 10ddf4f..9cae3c8 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,12 +1,13 @@ 0.1.10 ====== -:release date: NOT RELEASED YET +:release date: 2011-08-22 New feature ----------- * Add ``encoding`` and ``unicode_errors`` option to packer and unpacker. When this option is specified, (un)packs unicode object instead of bytes. - This enables using msgpack as a replacement of json. + This enables using msgpack as a replacement of json. (tailhook) + 0.1.9 ====== diff --git a/setup.py b/setup.py index 16987a4..0eeb84d 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 1, 9, 'final') +version = (0, 1, 10) import os import sys From a9483d860b0c7188cc409f2e94b0c022658d4dba Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 22 Aug 2011 03:37:14 +0900 Subject: [PATCH 0499/1172] Start 0.1.11a1.dev1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0eeb84d..b1a1221 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 1, 10) +version = (0, 1, 11, 'a1.dev1') import os import sys From 8361fd8da65bb1dd939edec390a6ced969413129 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 22 Aug 2011 03:38:16 +0900 Subject: [PATCH 0500/1172] Remove extra alias. --- msgpack/_msgpack.pyx | 2 -- 1 file changed, 2 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 5a83ea0..ae9b69f 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -168,7 +168,6 @@ def packb(object o, default=None, encoding='utf-8', unicode_errors='strict'): packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors) return packer.pack(o) -dumps = packs = packb cdef extern from "unpack.h": ctypedef struct msgpack_user: @@ -239,7 +238,6 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint else: return None -loads = unpacks = unpackb def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"): """unpack an object from stream.""" From 5ed2288fed84131b43e0ddeba34868b9d3a6a834 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 22 Aug 2011 03:41:24 +0900 Subject: [PATCH 0501/1172] Add signature to docstring. --- msgpack/_msgpack.pyx | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index ae9b69f..b004755 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -159,12 +159,18 @@ cdef class Packer(object): def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'): - """pack an object `o` and write it to stream).""" + """\ + pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict') + + pack an object `o` and write it to stream).""" packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors) stream.write(packer.pack(o)) def packb(object o, default=None, encoding='utf-8', unicode_errors='strict'): - """pack o and return packed bytes.""" + """\ + packb(object o, default=None, encoding='utf-8', unicode_errors='strict') + + pack o and return packed bytes.""" packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors) return packer.pack(o) @@ -191,7 +197,11 @@ cdef extern from "unpack.h": def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"): - """Unpack packed_bytes to object. Returns an unpacked object.""" + """\ + unpackb(object packed, object object_hook=None, object list_hook=None, + bint use_list=0, encoding=None, unicode_errors="strict") + + Unpack packed_bytes to object. Returns an unpacked object.""" cdef template_context ctx cdef size_t off = 0 cdef int ret @@ -240,7 +250,11 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"): - """unpack an object from stream.""" + """\ + unpack(object stream, object object_hook=None, object list_hook=None, + bint use_list=0, encoding=None, unicode_errors="strict") + + unpack an object from stream.""" return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors) From 938a1249739f1f049ef03d5a0a32336ba24f82e3 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 26 Aug 2011 04:45:05 +0900 Subject: [PATCH 0502/1172] Use cython's embedsignature directive and enhance some docstrings. --- msgpack/_msgpack.pyx | 49 +++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index b004755..9c2938c 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -1,4 +1,5 @@ # coding: utf-8 +#cython: embedsignature=True from cpython cimport * cdef extern from "Python.h": @@ -159,17 +160,13 @@ cdef class Packer(object): def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'): - """\ - pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict') - + """ pack an object `o` and write it to stream).""" packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors) stream.write(packer.pack(o)) def packb(object o, default=None, encoding='utf-8', unicode_errors='strict'): - """\ - packb(object o, default=None, encoding='utf-8', unicode_errors='strict') - + """ pack o and return packed bytes.""" packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors) return packer.pack(o) @@ -197,10 +194,7 @@ cdef extern from "unpack.h": def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"): - """\ - unpackb(object packed, object object_hook=None, object list_hook=None, - bint use_list=0, encoding=None, unicode_errors="strict") - + """ Unpack packed_bytes to object. Returns an unpacked object.""" cdef template_context ctx cdef size_t off = 0 @@ -250,21 +244,36 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"): - """\ - unpack(object stream, object object_hook=None, object list_hook=None, - bint use_list=0, encoding=None, unicode_errors="strict") - - unpack an object from stream.""" + """ + unpack an object from stream. + """ return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors) cdef class Unpacker(object): - """Unpacker(read_size=1024*1024) - + """ Streaming unpacker. read_size is used like file_like.read(read_size) - example: + `file_like` is a file-like object having `.read(n)` method. + When `Unpacker` initialized with `file_like`, unpacker reads serialized data + from it and `.feed()` method is not usable. + + `read_size` is used as `file_like.read(read_size)`. (default: 1M) + + If `use_list` is true, msgpack list is deserialized to Python list. + Otherwise, it is deserialized to Python tuple. (default: False) + + `object_hook` is same to simplejson. If it is not None, it should be callable + and Unpacker calls it when deserializing key-value. + + `encoding` is encoding used for decoding msgpack bytes. If it is None (default), + msgpack bytes is deserialized to Python bytes. + + `unicode_errors` is used for decoding bytes. + + example:: + unpacker = Unpacker() while 1: buf = astream.read() @@ -292,11 +301,9 @@ cdef class Unpacker(object): free(self.buf); self.buf = NULL; - def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, + def __init__(self, file_like=None, Py_ssize_t read_size=1024*1024, bint use_list=0, object object_hook=None, object list_hook=None, encoding=None, unicode_errors='strict'): - if read_size == 0: - read_size = 1024*1024 self.use_list = use_list self.file_like = file_like if file_like: From 2c53c1797975b06929e2fbb4e4b462aef502eacf Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 2 Sep 2011 16:16:08 +0900 Subject: [PATCH 0503/1172] Add test/ and test3/ to MANIFEST.in --- MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index cbc9ada..bf312d5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,5 @@ include setup.py include COPYING recursive-include msgpack *.h *.c *.pyx +recursive-include test *.py +recursive-include test3 *.py From 8312e986bfe5fea3c4d88259b06fa73d47130de4 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 2 Sep 2011 16:20:01 +0900 Subject: [PATCH 0504/1172] Update changelog --- ChangeLog.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 9cae3c8..9b407cd 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,13 @@ +0.1.11 +======= +:release date: NOT RELEASED YET + +Bugs fixed +------------- + +* Include test code for Python3 to sdist. (Johan Bergström) + + 0.1.10 ====== :release date: 2011-08-22 From d72bd0c5d4c09b78ce1912d0101709852573abc2 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 2 Sep 2011 16:22:21 +0900 Subject: [PATCH 0505/1172] Remove unnecessary semicolon. --- msgpack/_msgpack.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 9c2938c..7a81746 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -298,8 +298,8 @@ cdef class Unpacker(object): self.buf = NULL def __dealloc__(self): - free(self.buf); - self.buf = NULL; + free(self.buf) + self.buf = NULL def __init__(self, file_like=None, Py_ssize_t read_size=1024*1024, bint use_list=0, object object_hook=None, object list_hook=None, From 1f829173cd62d175721fa71da481becbece8826c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 25 Dec 2011 12:48:36 +0900 Subject: [PATCH 0506/1172] add *.so to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 35b00cc..1c29d6e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ build/* dist/* *.pyc *.pyo +*.so msgpack/__version__.py msgpack/_msgpack.c From b553754edf9597a11122956c567fd713e0537a3e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 25 Dec 2011 12:54:19 +0900 Subject: [PATCH 0507/1172] Fix compile error on MSVC. (davidgaleano) --- msgpack/pack.h | 4 ++++ msgpack/pack_template.h | 4 ++-- msgpack/unpack_template.h | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/msgpack/pack.h b/msgpack/pack.h index 2ae95d1..d36b436 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -25,6 +25,10 @@ extern "C" { #endif +#ifdef _MSC_VER +#define inline __inline +#endif + typedef struct msgpack_packer { char *buf; size_t length; diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index de148bf..4b50895 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -555,8 +555,8 @@ if(sizeof(unsigned long long) == 2) { msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { union { char buf[4]; uint32_t num; } f; - *((float*)&f.buf) = d; // FIXME unsigned char buf[5]; + *((float*)&f.buf) = d; // FIXME buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(f.num); msgpack_pack_append_buffer(x, buf, 5); } @@ -564,8 +564,8 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { union { char buf[8]; uint64_t num; } f; - *((double*)&f.buf) = d; // FIXME unsigned char buf[9]; + *((double*)&f.buf) = d; // FIXME buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(f.num); msgpack_pack_append_buffer(x, buf, 9); } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 7a2288f..42fe3a1 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -82,8 +82,6 @@ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) { - assert(len >= *off); - const unsigned char* p = (unsigned char*)data + *off; const unsigned char* const pe = (unsigned char*)data + len; const void* n = NULL; @@ -91,6 +89,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c unsigned int trail = ctx->trail; unsigned int cs = ctx->cs; unsigned int top = ctx->top; + msgpack_unpack_struct(_stack)* stack = ctx->stack; msgpack_unpack_user* user = &ctx->user; @@ -99,6 +98,8 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c int ret; + assert(len >= *off); + #define push_simple_value(func) \ if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \ goto _push From d2a23cc22a5998fb95a7d4c1bf1f6fbd17d27f3b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 26 Dec 2011 19:04:33 +0900 Subject: [PATCH 0508/1172] Release 0.1.11 --- ChangeLog.rst | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 9b407cd..758c83e 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,11 +1,12 @@ 0.1.11 ======= -:release date: NOT RELEASED YET +:release date: 2011-12-26 Bugs fixed ------------- * Include test code for Python3 to sdist. (Johan Bergström) +* Fix compilation error on MSVC. (davidgaleano) 0.1.10 diff --git a/setup.py b/setup.py index b1a1221..faeb9f1 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 1, 11, 'a1.dev1') +version = (0, 1, 11) import os import sys From ac713705ebdd131acb92b52116e6ec180731add2 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 27 Dec 2011 21:34:40 +0900 Subject: [PATCH 0509/1172] 0.1.12: re-enable packs()/unpacks() aliases. --- ChangeLog.rst | 10 ++++++++++ msgpack/__init__.py | 9 +++++++++ setup.py | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 758c83e..89a97d7 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,13 @@ +0.1.12 +======= +:release date: 2011-12-27 + +Bugs fixed +------------- + +* Re-enable packs/unpacks removed at 0.1.11. It will be removed when 0.2 is released. + + 0.1.11 ======= :release date: 2011-12-26 diff --git a/msgpack/__init__.py b/msgpack/__init__.py index cdf045f..6b9735e 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -9,3 +9,12 @@ loads = unpackb dump = pack dumps = packb +def packs(*args, **kw): + from warnings import warn + warn("msgpack.packs() is deprecated. Use packb() instead.", DeprecationWarning) + return packb(*args, **kw) + +def unpacks(*args, **kw): + from warnings import warn + warn("msgpack.unpacks() is deprecated. Use unpackb() instead.", DeprecationWarning) + return unpackb(*args, **kw) diff --git a/setup.py b/setup.py index faeb9f1..ddacbe2 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 1, 11) +version = (0, 1, 12) import os import sys From d685614138e588dc82b84cddae254fc120279762 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Wed, 18 Jan 2012 19:02:11 +0100 Subject: [PATCH 0510/1172] adopt setup.py to work with python installation older 2.7 --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ddacbe2..f34e341 100755 --- a/setup.py +++ b/setup.py @@ -45,7 +45,9 @@ else: Sdist = sdist -libraries = ['ws2_32'] if sys.platform == 'win32' else [] +libraries = [] +if sys.platform == 'win32': + libraries.append('ws2_32') msgpack_mod = Extension('msgpack._msgpack', sources=sources, From b76416977554c7b01442efe0d1f058a43372e2f5 Mon Sep 17 00:00:00 2001 From: David Wolever Date: Fri, 10 Feb 2012 15:08:49 -0500 Subject: [PATCH 0511/1172] Correcting 'utf-8' to 'unicode'. --- msgpack/_msgpack.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 7a81746..52464fb 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -121,7 +121,7 @@ cdef class Packer(object): ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyUnicode_Check(o): if not self.encoding: - raise TypeError("Can't encode utf-8 no encoding is specified") + raise TypeError("Can't encode unicode string: no encoding is specified") o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) rawval = o ret = msgpack_pack_raw(&self.pk, len(o)) @@ -425,3 +425,4 @@ cdef class Unpacker(object): #def _off(self): # return self.buf_head + From 5b878b6038ac7d8d180f359f728531e666717f94 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Thu, 16 Feb 2012 17:15:04 +0100 Subject: [PATCH 0512/1172] Be greedier when checking for tuples or lists. --- msgpack/_msgpack.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 52464fb..7f369a8 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -136,7 +136,7 @@ cdef class Packer(object): if ret != 0: break ret = self._pack(v, nest_limit-1) if ret != 0: break - elif PySequence_Check(o): + elif PyTuple_Check(o) or PyList_Check(o): ret = msgpack_pack_array(&self.pk, len(o)) if ret == 0: for v in o: From 31b7fda17b464176c34908da366bed5e4545823d Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 28 Feb 2012 15:36:58 +0100 Subject: [PATCH 0513/1172] Fix massive memory leak with object_hook and list_hook when unpacking. --- msgpack/unpack.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 0586ca8..6b443f2 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -161,8 +161,10 @@ static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_obj { if (u->list_hook) { PyObject *arglist = Py_BuildValue("(O)", *c); - *c = PyEval_CallObject(u->list_hook, arglist); + msgpack_unpack_object *new_c = PyEval_CallObject(u->list_hook, arglist); Py_DECREF(arglist); + Py_DECREF(*c); + *c = new_c; } return 0; } @@ -190,8 +192,10 @@ static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_objec { if (u->object_hook) { PyObject *arglist = Py_BuildValue("(O)", *c); - *c = PyEval_CallObject(u->object_hook, arglist); + msgpack_unpack_object *new_c = PyEval_CallObject(u->object_hook, arglist); Py_DECREF(arglist); + Py_DECREF(*c); + *c = new_c; } return 0; } From a5bc6b73856ca9c59f140427a26a6ffd5103b86f Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Tue, 28 Feb 2012 15:41:44 +0100 Subject: [PATCH 0514/1172] Better prototypes. --- msgpack/unpack.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 6b443f2..2659a97 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -161,7 +161,7 @@ static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_obj { if (u->list_hook) { PyObject *arglist = Py_BuildValue("(O)", *c); - msgpack_unpack_object *new_c = PyEval_CallObject(u->list_hook, arglist); + PyObject *new_c = PyEval_CallObject(u->list_hook, arglist); Py_DECREF(arglist); Py_DECREF(*c); *c = new_c; @@ -192,7 +192,7 @@ static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_objec { if (u->object_hook) { PyObject *arglist = Py_BuildValue("(O)", *c); - msgpack_unpack_object *new_c = PyEval_CallObject(u->object_hook, arglist); + PyObject *new_c = PyEval_CallObject(u->object_hook, arglist); Py_DECREF(arglist); Py_DECREF(*c); *c = new_c; From 64bdf6bcd633b4404cb5413f37a1c7ae428f048c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 8 Mar 2012 16:59:08 +0900 Subject: [PATCH 0515/1172] small optimization. --- msgpack/_msgpack.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 7f369a8..612dd0f 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -131,7 +131,7 @@ cdef class Packer(object): d = o ret = msgpack_pack_map(&self.pk, len(d)) if ret == 0: - for k,v in d.items(): + for k,v in d.iteritems(): ret = self._pack(k, nest_limit-1) if ret != 0: break ret = self._pack(v, nest_limit-1) @@ -149,7 +149,7 @@ cdef class Packer(object): raise TypeError("can't serialize %r" % (o,)) return ret - def pack(self, object obj): + cpdef pack(self, object obj): cdef int ret ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) if ret: From 7a4af28fa1599f49cc0a27c66beecd5e4474dcbe Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 21 Apr 2012 14:25:07 +0900 Subject: [PATCH 0516/1172] release 0.1.13 --- ChangeLog.rst | 21 +++++++++++++++++++++ setup.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 89a97d7..e0a4fd1 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,24 @@ +0.1.13 +======= +:release date: 2012-04-21 + +New +---- +* Don't accept subtype of list and tuple as msgpack list. (Steeve Morin) + It allows customize how it serialized with ``default`` argument. + +Bugs fixed +----------- +* Fix wrong error message. (David Wolever) +* Fix memory leak while unpacking when ``object_hook`` or ``list_hook`` is used. + (Steeve Morin) + +Other changes +------------- +* setup.py works on Python 2.5 (Steffen Siering) +* Optimization for serializing dict. + + 0.1.12 ======= :release date: 2011-12-27 diff --git a/setup.py b/setup.py index f34e341..b4b1771 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 1, 12) +version = (0, 1, 13) import os import sys From 07506667c98f51bf46b47116f9afed8fa5f381e0 Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Sat, 16 Jun 2012 13:56:46 +0300 Subject: [PATCH 0517/1172] Make sure objects inherited from Dict are properly casted (or else Cython will complain and crash). --- msgpack/_msgpack.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 612dd0f..990f585 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -128,7 +128,7 @@ cdef class Packer(object): if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyDict_Check(o): - d = o + d = o ret = msgpack_pack_map(&self.pk, len(d)) if ret == 0: for k,v in d.iteritems(): From 40d4b8946b5e25045eb1047f8d0d3d4c5ee71282 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 19 Jun 2012 13:39:23 +0900 Subject: [PATCH 0518/1172] Start using tox --- tox.ini | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tox.ini diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..c0a5629 --- /dev/null +++ b/tox.ini @@ -0,0 +1,9 @@ +[tox] +envlist = py26,py27,py32 +[testenv] +deps= + nose +commands=nosetests -w test + +[testenv:py32] +commands=nosetests -w test3 From f1dd03fe80de1c2b267d79bd3d74dec2592a2339 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 19 Jun 2012 13:39:32 +0900 Subject: [PATCH 0519/1172] Add test for subtype. --- test/test_subtype.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 test/test_subtype.py diff --git a/test/test_subtype.py b/test/test_subtype.py new file mode 100644 index 0000000..1dfd7da --- /dev/null +++ b/test/test_subtype.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * +from msgpack import packb, unpackb +from collections import namedtuple + +class MyList(list): + pass + +class MyDict(dict): + pass + +class MyTuple(tuple): + pass + +MyNamedTuple = namedtuple('MyNamedTuple', 'x y') + +def test_types(): + assert_equal(packb(dict()), packb(MyDict())) + assert_equal(packb(list()), packb(MyList())) + assert_equal(packb(MyNamedTuple(1,2)), packb((1,2))) + + +if __name__ == '__main__': + main() From 76f34667a02f461043a70d776ec05fc1f90bd9e9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 19 Jun 2012 13:40:00 +0900 Subject: [PATCH 0520/1172] Add ".tox" to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1c29d6e..9452edd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ MANIFEST build/* dist/* +.tox *.pyc *.pyo *.so From 0b38e86534130f625cbea2f9446e8e52ef5f5a06 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 19 Jun 2012 13:55:14 +0900 Subject: [PATCH 0521/1172] unify tests for py2 and py3 --- test/test_buffer.py | 4 +- test/test_case.py | 40 +++++++-------- test/test_format.py | 50 +++++++++---------- test/test_obj.py | 15 +++--- test/test_pack.py | 34 ++++++------- test/test_sequnpack.py | 28 ++++++----- test3/test_buffer.py | 16 ------ test3/test_case.py | 105 ---------------------------------------- test3/test_except.py | 14 ------ test3/test_format.py | 75 ---------------------------- test3/test_obj.py | 44 ----------------- test3/test_pack.py | 82 ------------------------------- test3/test_sequnpack.py | 36 -------------- tox.ini | 4 +- 14 files changed, 85 insertions(+), 462 deletions(-) delete mode 100644 test3/test_buffer.py delete mode 100644 test3/test_case.py delete mode 100644 test3/test_except.py delete mode 100644 test3/test_format.py delete mode 100644 test3/test_obj.py delete mode 100644 test3/test_pack.py delete mode 100644 test3/test_sequnpack.py diff --git a/test/test_buffer.py b/test/test_buffer.py index ce7a72d..01310a0 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -7,10 +7,10 @@ from msgpack import packb, unpackb def test_unpack_buffer(): from array import array - buf = array('c') + buf = array('b') buf.fromstring(packb(('foo', 'bar'))) obj = unpackb(buf) - assert_equal(('foo', 'bar'), obj) + assert_equal((b'foo', b'bar'), obj) if __name__ == '__main__': main() diff --git a/test/test_case.py b/test/test_case.py index 1cbc494..2f42316 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -40,7 +40,7 @@ def test_9(): def check_raw(overhead, num): - check(num + overhead, " " * num) + check(num + overhead, b" " * num) def test_fixraw(): check_raw(1, 0) @@ -75,31 +75,31 @@ def match(obj, buf): def test_match(): cases = [ - (None, '\xc0'), - (False, '\xc2'), - (True, '\xc3'), - (0, '\x00'), - (127, '\x7f'), - (128, '\xcc\x80'), - (256, '\xcd\x01\x00'), - (-1, '\xff'), - (-33, '\xd0\xdf'), - (-129, '\xd1\xff\x7f'), - ({1:1}, '\x81\x01\x01'), - (1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"), - ((), '\x90'), - (tuple(range(15)),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), - (tuple(range(16)),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), - ({}, '\x80'), - (dict([(x,x) for x in range(15)]), '\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'), - (dict([(x,x) for x in range(16)]), '\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'), + (None, b'\xc0'), + (False, b'\xc2'), + (True, b'\xc3'), + (0, b'\x00'), + (127, b'\x7f'), + (128, b'\xcc\x80'), + (256, b'\xcd\x01\x00'), + (-1, b'\xff'), + (-33, b'\xd0\xdf'), + (-129, b'\xd1\xff\x7f'), + ({1:1}, b'\x81\x01\x01'), + (1.0, b"\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"), + ((), b'\x90'), + (tuple(range(15)),b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), + (tuple(range(16)),b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), + ({}, b'\x80'), + (dict([(x,x) for x in range(15)]), b'\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'), + (dict([(x,x) for x in range(16)]), b'\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'), ] for v, p in cases: match(v, p) def test_unicode(): - assert_equal('foobar', unpacks(packs(u'foobar'))) + assert_equal(b'foobar', unpacks(packs('foobar'))) if __name__ == '__main__': main() diff --git a/test/test_format.py b/test/test_format.py index 562ef54..022e680 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -9,65 +9,65 @@ def check(src, should): assert_equal(unpacks(src), should) def testSimpleValue(): - check("\x93\xc0\xc2\xc3", + check(b"\x93\xc0\xc2\xc3", (None, False, True,)) def testFixnum(): - check("\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", + check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", ((0,64,127,), (-32,-16,-1,),) ) def testFixArray(): - check("\x92\x90\x91\x91\xc0", + check(b"\x92\x90\x91\x91\xc0", ((),((None,),),), ) def testFixRaw(): - check("\x94\xa0\xa1a\xa2bc\xa3def", - ("", "a", "bc", "def",), + check(b"\x94\xa0\xa1a\xa2bc\xa3def", + (b"", b"a", b"bc", b"def",), ) def testFixMap(): check( - "\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", + b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", {False: {None: None}, True:{None:{}}}, ) def testUnsignedInt(): check( - "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" - "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" - "\xce\xff\xff\xff\xff", + b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" + b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" + b"\xce\xff\xff\xff\xff", (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,), ) def testSignedInt(): - check("\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" - "\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" - "\xd2\xff\xff\xff\xff", + check(b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" + b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" + b"\xd2\xff\xff\xff\xff", (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,)) def testRaw(): - check("\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" - "\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", - ("", "a", "ab", "", "a", "ab")) + check(b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" + b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", + (b"", b"a", b"ab", b"", b"a", b"ab")) def testArray(): - check("\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" - "\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02" - "\xc2\xc3", + check(b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" + b"\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02" + b"\xc2\xc3", ((), (None,), (False,True), (), (None,), (False,True)) ) def testMap(): check( - "\x96" - "\xde\x00\x00" - "\xde\x00\x01\xc0\xc2" - "\xde\x00\x02\xc0\xc2\xc3\xc2" - "\xdf\x00\x00\x00\x00" - "\xdf\x00\x00\x00\x01\xc0\xc2" - "\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", + b"\x96" + b"\xde\x00\x00" + b"\xde\x00\x01\xc0\xc2" + b"\xde\x00\x02\xc0\xc2\xc3\xc2" + b"\xdf\x00\x00\x00\x00" + b"\xdf\x00\x00\x00\x01\xc0\xc2" + b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", ({}, {None: False}, {True: False, None: False}, {}, {None: False}, {True: False, None: False})) diff --git a/test/test_obj.py b/test/test_obj.py index 0eebe7b..6357cfc 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -7,22 +7,22 @@ from nose.tools import * from msgpack import packs, unpacks def _decode_complex(obj): - if '__complex__' in obj: - return complex(obj['real'], obj['imag']) + if b'__complex__' in obj: + return complex(obj[b'real'], obj[b'imag']) return obj def _encode_complex(obj): if isinstance(obj, complex): - return {'__complex__': True, 'real': 1, 'imag': 2} + return {b'__complex__': True, b'real': 1, b'imag': 2} return obj def test_encode_hook(): packed = packs([3, 1+2j], default=_encode_complex) unpacked = unpacks(packed) - eq_(unpacked[1], {'__complex__': True, 'real': 1, 'imag': 2}) + eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2}) def test_decode_hook(): - packed = packs([3, {'__complex__': True, 'real': 1, 'imag': 2}]) + packed = packs([3, {b'__complex__': True, b'real': 1, b'imag': 2}]) unpacked = unpacks(packed, object_hook=_decode_complex) eq_(unpacked[1], 1+2j) @@ -40,7 +40,4 @@ def test_array_hook(): eq_(unpacked, '123') if __name__ == '__main__': - test_decode_hook() - test_encode_hook() - test_bad_hook() - test_array_hook() + main() diff --git a/test/test_pack.py b/test/test_pack.py index 2e1ac22..480af28 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -5,9 +5,9 @@ from nose import main from nose.tools import * from nose.plugins.skip import SkipTest -from msgpack import packs, unpacks, Packer, Unpacker +from msgpack import packs, unpacks, Unpacker, Packer -from StringIO import StringIO +from io import BytesIO def check(data): re = unpacks(packs(data)) @@ -18,7 +18,7 @@ def testPack(): 0, 1, 127, 128, 255, 256, 65535, 65536, -1, -32, -33, -128, -129, -32768, -32769, 1.0, - "", "a", "a"*31, "a"*32, + b"", b"a", b"a"*31, b"a"*32, None, True, False, (), ((),), ((), None,), {None: 0}, @@ -29,20 +29,20 @@ def testPack(): def testPackUnicode(): test_data = [ - u"", u"abcd", (u"defgh",), u"РуÑÑкий текÑÑ‚", + "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚", ] for td in test_data: re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8') assert_equal(re, td) packer = Packer(encoding='utf-8') data = packer.pack(td) - re = Unpacker(StringIO(data), encoding='utf-8').unpack() + re = Unpacker(BytesIO(data), encoding='utf-8').unpack() assert_equal(re, td) def testPackUTF32(): try: test_data = [ - u"", u"abcd", (u"defgh",), u"РуÑÑкий текÑÑ‚", + "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚", ] for td in test_data: re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32') @@ -52,37 +52,35 @@ def testPackUTF32(): def testPackBytes(): test_data = [ - "", "abcd", ("defgh",), + b"", b"abcd", (b"defgh",), ] for td in test_data: check(td) def testIgnoreUnicodeErrors(): - re = unpacks(packs('abc\xeddef'), - encoding='ascii', unicode_errors='ignore') + re = unpacks(packs(b'abc\xeddef'), + encoding='utf-8', unicode_errors='ignore') assert_equal(re, "abcdef") @raises(UnicodeDecodeError) def testStrictUnicodeUnpack(): - unpacks(packs('abc\xeddef'), encoding='utf-8') + unpacks(packs(b'abc\xeddef'), encoding='utf-8') @raises(UnicodeEncodeError) def testStrictUnicodePack(): - packs(u"abc\xeddef", encoding='ascii', unicode_errors='strict') + packs("abc\xeddef", encoding='ascii', unicode_errors='strict') def testIgnoreErrorsPack(): - re = unpacks( - packs(u"abcФФФdef", encoding='ascii', unicode_errors='ignore'), - encoding='utf-8') - assert_equal(re, u"abcdef") + re = unpacks(packs("abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8') + assert_equal(re, "abcdef") @raises(TypeError) def testNoEncoding(): - packs(u"abc", encoding=None) + packs("abc", encoding=None) def testDecodeBinary(): - re = unpacks(packs(u"abc"), encoding=None) - assert_equal(re, "abc") + re = unpacks(packs("abc"), encoding=None) + assert_equal(re, b"abc") if __name__ == '__main__': main() diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index d61be23..5fd377c 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,33 +1,35 @@ #!/usr/bin/env python # coding: utf-8 + + from msgpack import Unpacker def test_foobar(): unpacker = Unpacker(read_size=3) - unpacker.feed('foobar') - assert unpacker.unpack() == ord('f') - assert unpacker.unpack() == ord('o') - assert unpacker.unpack() == ord('o') - assert unpacker.unpack() == ord('b') - assert unpacker.unpack() == ord('a') - assert unpacker.unpack() == ord('r') + unpacker.feed(b'foobar') + assert unpacker.unpack() == ord(b'f') + assert unpacker.unpack() == ord(b'o') + assert unpacker.unpack() == ord(b'o') + assert unpacker.unpack() == ord(b'b') + assert unpacker.unpack() == ord(b'a') + assert unpacker.unpack() == ord(b'r') try: o = unpacker.unpack() - print "Oops!", o + print(("Oops!", o)) assert 0 except StopIteration: assert 1 else: assert 0 - unpacker.feed('foo') - unpacker.feed('bar') + unpacker.feed(b'foo') + unpacker.feed(b'bar') k = 0 - for o, e in zip(unpacker, 'foobarbaz'): - assert o == ord(e) + for o, e in zip(unpacker, b'foobarbaz'): + assert o == e k += 1 - assert k == len('foobar') + assert k == len(b'foobar') if __name__ == '__main__': test_foobar() diff --git a/test3/test_buffer.py b/test3/test_buffer.py deleted file mode 100644 index 01310a0..0000000 --- a/test3/test_buffer.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -from nose import main -from nose.tools import * -from msgpack import packb, unpackb - -def test_unpack_buffer(): - from array import array - buf = array('b') - buf.fromstring(packb(('foo', 'bar'))) - obj = unpackb(buf) - assert_equal((b'foo', b'bar'), obj) - -if __name__ == '__main__': - main() diff --git a/test3/test_case.py b/test3/test_case.py deleted file mode 100644 index 2f42316..0000000 --- a/test3/test_case.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -from nose import main -from nose.tools import * -from msgpack import packs, unpacks - - -def check(length, obj): - v = packs(obj) - assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) - assert_equal(unpacks(v), obj) - -def test_1(): - for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1, - -((1<<5)-1), -(1<<5)]: - check(1, o) - -def test_2(): - for o in [1 << 7, (1 << 8) - 1, - -((1<<5)+1), -(1<<7) - ]: - check(2, o) - -def test_3(): - for o in [1 << 8, (1 << 16) - 1, - -((1<<7)+1), -(1<<15)]: - check(3, o) - -def test_5(): - for o in [1 << 16, (1 << 32) - 1, - -((1<<15)+1), -(1<<31)]: - check(5, o) - -def test_9(): - for o in [1 << 32, (1 << 64) - 1, - -((1<<31)+1), -(1<<63), - 1.0, 0.1, -0.1, -1.0]: - check(9, o) - - -def check_raw(overhead, num): - check(num + overhead, b" " * num) - -def test_fixraw(): - check_raw(1, 0) - check_raw(1, (1<<5) - 1) - -def test_raw16(): - check_raw(3, 1<<5) - check_raw(3, (1<<16) - 1) - -def test_raw32(): - check_raw(5, 1<<16) - - -def check_array(overhead, num): - check(num + overhead, (None,) * num) - -def test_fixarray(): - check_array(1, 0) - check_array(1, (1 << 4) - 1) - -def test_array16(): - check_array(3, 1 << 4) - check_array(3, (1<<16)-1) - -def test_array32(): - check_array(5, (1<<16)) - - -def match(obj, buf): - assert_equal(packs(obj), buf) - assert_equal(unpacks(buf), obj) - -def test_match(): - cases = [ - (None, b'\xc0'), - (False, b'\xc2'), - (True, b'\xc3'), - (0, b'\x00'), - (127, b'\x7f'), - (128, b'\xcc\x80'), - (256, b'\xcd\x01\x00'), - (-1, b'\xff'), - (-33, b'\xd0\xdf'), - (-129, b'\xd1\xff\x7f'), - ({1:1}, b'\x81\x01\x01'), - (1.0, b"\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"), - ((), b'\x90'), - (tuple(range(15)),b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), - (tuple(range(16)),b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), - ({}, b'\x80'), - (dict([(x,x) for x in range(15)]), b'\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'), - (dict([(x,x) for x in range(16)]), b'\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'), - ] - - for v, p in cases: - match(v, p) - -def test_unicode(): - assert_equal(b'foobar', unpacks(packs('foobar'))) - -if __name__ == '__main__': - main() diff --git a/test3/test_except.py b/test3/test_except.py deleted file mode 100644 index 574728f..0000000 --- a/test3/test_except.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -from nose.tools import * -from msgpack import packs, unpacks - -import datetime - -def test_raise_on_find_unsupported_value(): - assert_raises(TypeError, packs, datetime.datetime.now()) - -if __name__ == '__main__': - from nose import main - main() diff --git a/test3/test_format.py b/test3/test_format.py deleted file mode 100644 index 022e680..0000000 --- a/test3/test_format.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -from nose import main -from nose.tools import * -from msgpack import unpacks - -def check(src, should): - assert_equal(unpacks(src), should) - -def testSimpleValue(): - check(b"\x93\xc0\xc2\xc3", - (None, False, True,)) - -def testFixnum(): - check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", - ((0,64,127,), (-32,-16,-1,),) - ) - -def testFixArray(): - check(b"\x92\x90\x91\x91\xc0", - ((),((None,),),), - ) - -def testFixRaw(): - check(b"\x94\xa0\xa1a\xa2bc\xa3def", - (b"", b"a", b"bc", b"def",), - ) - -def testFixMap(): - check( - b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", - {False: {None: None}, True:{None:{}}}, - ) - -def testUnsignedInt(): - check( - b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" - b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" - b"\xce\xff\xff\xff\xff", - (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,), - ) - -def testSignedInt(): - check(b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" - b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" - b"\xd2\xff\xff\xff\xff", - (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,)) - -def testRaw(): - check(b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" - b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", - (b"", b"a", b"ab", b"", b"a", b"ab")) - -def testArray(): - check(b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" - b"\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02" - b"\xc2\xc3", - ((), (None,), (False,True), (), (None,), (False,True)) - ) - -def testMap(): - check( - b"\x96" - b"\xde\x00\x00" - b"\xde\x00\x01\xc0\xc2" - b"\xde\x00\x02\xc0\xc2\xc3\xc2" - b"\xdf\x00\x00\x00\x00" - b"\xdf\x00\x00\x00\x01\xc0\xc2" - b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", - ({}, {None: False}, {True: False, None: False}, {}, - {None: False}, {True: False, None: False})) - -if __name__ == '__main__': - main() diff --git a/test3/test_obj.py b/test3/test_obj.py deleted file mode 100644 index b54021f..0000000 --- a/test3/test_obj.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -from nose import main -from nose.tools import * - -from msgpack import packs, unpacks - -def _decode_complex(obj): - if b'__complex__' in obj: - return complex(obj[b'real'], obj[b'imag']) - return obj - -def _encode_complex(obj): - if isinstance(obj, complex): - return {b'__complex__': True, b'real': 1, b'imag': 2} - return obj - -def test_encode_hook(): - packed = packs([3, 1+2j], default=_encode_complex) - unpacked = unpacks(packed) - eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2}) - -def test_decode_hook(): - packed = packs([3, {b'__complex__': True, b'real': 1, b'imag': 2}]) - unpacked = unpacks(packed, object_hook=_decode_complex) - eq_(unpacked[1], 1+2j) - -@raises(ValueError) -def test_bad_hook(): - packed = packs([3, 1+2j], default=lambda o: o) - unpacked = unpacks(packed) - -def _arr_to_str(arr): - return ''.join(str(c) for c in arr) - -def test_array_hook(): - packed = packs([1,2,3]) - unpacked = unpacks(packed, list_hook=_arr_to_str) - eq_(unpacked, '123') - -if __name__ == '__main__': - #main() - test_decode_hook() diff --git a/test3/test_pack.py b/test3/test_pack.py deleted file mode 100644 index 5ff04e7..0000000 --- a/test3/test_pack.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -from nose import main -from nose.tools import * - -from msgpack import packs, unpacks, Unpacker, Packer - -from io import BytesIO - -def check(data): - re = unpacks(packs(data)) - assert_equal(re, data) - -def testPack(): - test_data = [ - 0, 1, 127, 128, 255, 256, 65535, 65536, - -1, -32, -33, -128, -129, -32768, -32769, - 1.0, - b"", b"a", b"a"*31, b"a"*32, - None, True, False, - (), ((),), ((), None,), - {None: 0}, - (1<<23), - ] - for td in test_data: - check(td) - -def testPackUnicode(): - test_data = [ - "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚", - ] - for td in test_data: - re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8') - assert_equal(re, td) - packer = Packer(encoding='utf-8') - data = packer.pack(td) - re = Unpacker(BytesIO(data), encoding='utf-8').unpack() - assert_equal(re, td) - -def testPackUTF32(): - test_data = [ - "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚", - ] - for td in test_data: - re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32') - assert_equal(re, td) - -def testPackBytes(): - test_data = [ - b"", b"abcd", (b"defgh",), - ] - for td in test_data: - check(td) - -def testIgnoreUnicodeErrors(): - re = unpacks(packs(b'abc\xeddef'), - encoding='utf-8', unicode_errors='ignore') - assert_equal(re, "abcdef") - -@raises(UnicodeDecodeError) -def testStrictUnicodeUnpack(): - unpacks(packs(b'abc\xeddef'), encoding='utf-8') - -@raises(UnicodeEncodeError) -def testStrictUnicodePack(): - packs("abc\xeddef", encoding='ascii', unicode_errors='strict') - -def testIgnoreErrorsPack(): - re = unpacks(packs("abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8') - assert_equal(re, "abcdef") - -@raises(TypeError) -def testNoEncoding(): - packs("abc", encoding=None) - -def testDecodeBinary(): - re = unpacks(packs("abc"), encoding=None) - assert_equal(re, b"abc") - -if __name__ == '__main__': - main() diff --git a/test3/test_sequnpack.py b/test3/test_sequnpack.py deleted file mode 100644 index 5fd377c..0000000 --- a/test3/test_sequnpack.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - - - -from msgpack import Unpacker - -def test_foobar(): - unpacker = Unpacker(read_size=3) - unpacker.feed(b'foobar') - assert unpacker.unpack() == ord(b'f') - assert unpacker.unpack() == ord(b'o') - assert unpacker.unpack() == ord(b'o') - assert unpacker.unpack() == ord(b'b') - assert unpacker.unpack() == ord(b'a') - assert unpacker.unpack() == ord(b'r') - try: - o = unpacker.unpack() - print(("Oops!", o)) - assert 0 - except StopIteration: - assert 1 - else: - assert 0 - unpacker.feed(b'foo') - unpacker.feed(b'bar') - - k = 0 - for o, e in zip(unpacker, b'foobarbaz'): - assert o == e - k += 1 - assert k == len(b'foobar') - -if __name__ == '__main__': - test_foobar() - diff --git a/tox.ini b/tox.ini index c0a5629..276e16a 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,5 @@ envlist = py26,py27,py32 [testenv] deps= nose -commands=nosetests -w test -[testenv:py32] -commands=nosetests -w test3 +commands=nosetests -w test From 636f4529aa8426f4ee3e25e0b4931bb37384df1e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 19 Jun 2012 14:20:56 +0900 Subject: [PATCH 0522/1172] Fix tests to pass. --- test/test_pack.py | 16 ++++++++++------ test/test_seq.py | 19 ++++++++++--------- test/test_sequnpack.py | 14 +++++--------- tox.ini | 1 + 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/test/test_pack.py b/test/test_pack.py index 480af28..88b6dac 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # coding: utf-8 +import six from nose import main from nose.tools import * from nose.plugins.skip import SkipTest @@ -29,7 +30,7 @@ def testPack(): def testPackUnicode(): test_data = [ - "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚", + six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("РуÑÑкий текÑÑ‚"), ] for td in test_data: re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8') @@ -42,7 +43,10 @@ def testPackUnicode(): def testPackUTF32(): try: test_data = [ - "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚", + six.u(""), + six.u("abcd"), + (six.u("defgh"),), + six.u("РуÑÑкий текÑÑ‚"), ] for td in test_data: re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32') @@ -68,15 +72,15 @@ def testStrictUnicodeUnpack(): @raises(UnicodeEncodeError) def testStrictUnicodePack(): - packs("abc\xeddef", encoding='ascii', unicode_errors='strict') + packs(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict') def testIgnoreErrorsPack(): - re = unpacks(packs("abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8') - assert_equal(re, "abcdef") + re = unpacks(packs(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8') + assert_equal(re, six.u("abcdef")) @raises(TypeError) def testNoEncoding(): - packs("abc", encoding=None) + packs(six.u("abc"), encoding=None) def testDecodeBinary(): re = unpacks(packs("abc"), encoding=None) diff --git a/test/test_seq.py b/test/test_seq.py index 993a59e..d0f9ccc 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -1,21 +1,22 @@ #!/usr/bin/env python # coding: utf-8 +import six from nose import main from nose.tools import * -import StringIO +import io import msgpack -binarydata = [chr(i) for i in xrange(256)] -binarydata = "".join(binarydata) +binarydata = [chr(i) for i in range(256)] +binarydata = six.b("".join(binarydata)) def gen_binary_data(idx): data = binarydata[:idx % 300] return data def test_exceeding_unpacker_read_size(): - dumpf = StringIO.StringIO() + dumpf = io.BytesIO() packer = msgpack.Packer() @@ -26,18 +27,18 @@ def test_exceeding_unpacker_read_size(): # 40 ok for read_size=1024, while 50 introduces errors # 7000 ok for read_size=1024*1024, while 8000 leads to glibc detected *** python: double free or corruption (!prev): - for idx in xrange(NUMBER_OF_STRINGS): + for idx in range(NUMBER_OF_STRINGS): data = gen_binary_data(idx) dumpf.write(packer.pack(data)) - f = StringIO.StringIO(dumpf.getvalue()) + f = io.BytesIO(dumpf.getvalue()) dumpf.close() unpacker = msgpack.Unpacker(f, read_size=read_size) read_count = 0 for idx, o in enumerate(unpacker): - assert_equal(type(o), str) + assert_equal(type(o), bytes) assert_equal(o, gen_binary_data(idx)) read_count += 1 @@ -45,5 +46,5 @@ def test_exceeding_unpacker_read_size(): if __name__ == '__main__': - # main() - test_exceeding_unpacker_read_size() + main() + #test_exceeding_unpacker_read_size() diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 5fd377c..774fe1b 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,8 +1,6 @@ #!/usr/bin/env python # coding: utf-8 - - from msgpack import Unpacker def test_foobar(): @@ -16,18 +14,16 @@ def test_foobar(): assert unpacker.unpack() == ord(b'r') try: o = unpacker.unpack() - print(("Oops!", o)) - assert 0 + assert 0, "should raise exception" except StopIteration: - assert 1 - else: - assert 0 + assert 1, "ok" + unpacker.feed(b'foo') unpacker.feed(b'bar') k = 0 - for o, e in zip(unpacker, b'foobarbaz'): - assert o == e + for o, e in zip(unpacker, 'foobarbaz'): + assert o == ord(e) k += 1 assert k == len(b'foobar') diff --git a/tox.ini b/tox.ini index 276e16a..5e80dd0 100644 --- a/tox.ini +++ b/tox.ini @@ -3,5 +3,6 @@ envlist = py26,py27,py32 [testenv] deps= nose + six commands=nosetests -w test From 5b0353eac610a23742d80f2f243884364e14f67d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 19 Jun 2012 14:31:32 +0900 Subject: [PATCH 0523/1172] Start 0.1.14dev --- ChangeLog.rst | 13 +++++++++++++ setup.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index e0a4fd1..3a9277c 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,16 @@ +0.1.14 +======= +:release date: NOT RELEASED YET + +Changes +------- +* Drop supporting Python 2.5 and unify tests for Py2 and Py3. + +Bugs fixed +---------- +* #8 Packing subclass of dict raises TypeError. (Thanks to Steeve Morin.) + + 0.1.13 ======= :release date: 2012-04-21 diff --git a/setup.py b/setup.py index b4b1771..115d3b8 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 1, 13) +version = (0, 1, 14, 'dev') import os import sys From 812c8bcff42a5b77f9ddfce70289bf754cca108c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 26 Jun 2012 13:21:10 +0900 Subject: [PATCH 0524/1172] Update msgpack version. --- msgpack/pack_define.h | 5 +- msgpack/pack_template.h | 247 +++++++++++++++++++++++++------------- msgpack/sysdep.h | 137 ++++++++++++++++++--- msgpack/unpack_define.h | 9 +- msgpack/unpack_template.h | 127 ++++++++++++-------- 5 files changed, 370 insertions(+), 155 deletions(-) diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h index f72391b..4845d52 100644 --- a/msgpack/pack_define.h +++ b/msgpack/pack_define.h @@ -1,7 +1,7 @@ /* * MessagePack unpacking routine template * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,8 +18,9 @@ #ifndef MSGPACK_PACK_DEFINE_H__ #define MSGPACK_PACK_DEFINE_H__ -#include "sysdep.h" +#include "msgpack/sysdep.h" #include +#include #endif /* msgpack/pack_define.h */ diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 4b50895..65c959d 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -1,7 +1,7 @@ /* * MessagePack packing routine template * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,12 @@ * limitations under the License. */ -#ifdef __LITTLE_ENDIAN__ +#if defined(__LITTLE_ENDIAN__) #define TAKE8_8(d) ((uint8_t*)&d)[0] #define TAKE8_16(d) ((uint8_t*)&d)[0] #define TAKE8_32(d) ((uint8_t*)&d)[0] #define TAKE8_64(d) ((uint8_t*)&d)[0] -#elif __BIG_ENDIAN__ +#elif defined(__BIG_ENDIAN__) #define TAKE8_8(d) ((uint8_t*)&d)[0] #define TAKE8_16(d) ((uint8_t*)&d)[1] #define TAKE8_32(d) ((uint8_t*)&d)[3] @@ -69,7 +69,7 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -89,12 +89,12 @@ do { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -103,7 +103,7 @@ do { \ #define msgpack_pack_real_uint64(x, d) \ do { \ if(d < (1ULL<<8)) { \ - if(d < (1<<7)) { \ + if(d < (1ULL<<7)) { \ /* fixnum */ \ msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ @@ -113,19 +113,19 @@ do { \ } \ } else { \ if(d < (1ULL<<16)) { \ - /* signed 16 */ \ + /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ - /* signed 32 */ \ + /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ - /* signed 64 */ \ + /* unsigned 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ + buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -149,7 +149,7 @@ do { \ if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -167,7 +167,7 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -179,12 +179,12 @@ do { \ if(d < -(1<<15)) { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -202,12 +202,12 @@ do { \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -220,19 +220,19 @@ do { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ + buf[0] = 0xd3; _msgpack_store64(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -252,19 +252,19 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ + buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -272,63 +272,63 @@ do { \ } while(0) -#ifdef msgpack_pack_inline_func_fastint +#ifdef msgpack_pack_inline_func_fixint -msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) +msgpack_pack_inline_func_fixint(_uint8)(msgpack_pack_user x, uint8_t d) { unsigned char buf[2] = {0xcc, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } -msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) +msgpack_pack_inline_func_fixint(_uint16)(msgpack_pack_user x, uint16_t d) { unsigned char buf[3]; - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); + buf[0] = 0xcd; _msgpack_store16(&buf[1], d); msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) +msgpack_pack_inline_func_fixint(_uint32)(msgpack_pack_user x, uint32_t d) { unsigned char buf[5]; - buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); + buf[0] = 0xce; _msgpack_store32(&buf[1], d); msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) +msgpack_pack_inline_func_fixint(_uint64)(msgpack_pack_user x, uint64_t d) { unsigned char buf[9]; - buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); + buf[0] = 0xcf; _msgpack_store64(&buf[1], d); msgpack_pack_append_buffer(x, buf, 9); } -msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) +msgpack_pack_inline_func_fixint(_int8)(msgpack_pack_user x, int8_t d) { unsigned char buf[2] = {0xd0, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } -msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) +msgpack_pack_inline_func_fixint(_int16)(msgpack_pack_user x, int16_t d) { unsigned char buf[3]; - buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); + buf[0] = 0xd1; _msgpack_store16(&buf[1], d); msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) +msgpack_pack_inline_func_fixint(_int32)(msgpack_pack_user x, int32_t d) { unsigned char buf[5]; - buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); + buf[0] = 0xd2; _msgpack_store32(&buf[1], d); msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) +msgpack_pack_inline_func_fixint(_int64)(msgpack_pack_user x, int64_t d) { unsigned char buf[9]; - buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); + buf[0] = 0xd3; _msgpack_store64(&buf[1], d); msgpack_pack_append_buffer(x, buf, 9); } -#undef msgpack_pack_inline_func_fastint +#undef msgpack_pack_inline_func_fixint #endif @@ -377,14 +377,24 @@ msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d) msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d) { -#if defined(SIZEOF_SHORT) || defined(SHRT_MAX) -#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff +#if defined(SIZEOF_SHORT) +#if SIZEOF_SHORT == 2 msgpack_pack_real_int16(x, d); -#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff +#elif SIZEOF_SHORT == 4 msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif + +#elif defined(SHRT_MAX) +#if SHRT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SHRT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif + #else if(sizeof(short) == 2) { msgpack_pack_real_int16(x, d); @@ -398,14 +408,24 @@ if(sizeof(short) == 2) { msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d) { -#if defined(SIZEOF_INT) || defined(INT_MAX) -#if SIZEOF_INT == 2 || INT_MAX == 0x7fff +#if defined(SIZEOF_INT) +#if SIZEOF_INT == 2 msgpack_pack_real_int16(x, d); -#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff +#elif SIZEOF_INT == 4 msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif + +#elif defined(INT_MAX) +#if INT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif INT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif + #else if(sizeof(int) == 2) { msgpack_pack_real_int16(x, d); @@ -419,14 +439,24 @@ if(sizeof(int) == 2) { msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d) { -#if defined(SIZEOF_LONG) || defined(LONG_MAX) -#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL +#if defined(SIZEOF_LONG) +#if SIZEOF_LONG == 2 msgpack_pack_real_int16(x, d); -#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL +#elif SIZEOF_LONG == 4 msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif + +#elif defined(LONG_MAX) +#if LONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif LONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif + #else if(sizeof(long) == 2) { msgpack_pack_real_int16(x, d); @@ -440,14 +470,24 @@ if(sizeof(long) == 2) { msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d) { -#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX) -#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL +#if defined(SIZEOF_LONG_LONG) +#if SIZEOF_LONG_LONG == 2 msgpack_pack_real_int16(x, d); -#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL +#elif SIZEOF_LONG_LONG == 4 msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif + +#elif defined(LLONG_MAX) +#if LLONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif LLONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif + #else if(sizeof(long long) == 2) { msgpack_pack_real_int16(x, d); @@ -461,14 +501,24 @@ if(sizeof(long long) == 2) { msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d) { -#if defined(SIZEOF_SHORT) || defined(USHRT_MAX) -#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU +#if defined(SIZEOF_SHORT) +#if SIZEOF_SHORT == 2 msgpack_pack_real_uint16(x, d); -#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU +#elif SIZEOF_SHORT == 4 msgpack_pack_real_uint32(x, d); #else msgpack_pack_real_uint64(x, d); #endif + +#elif defined(USHRT_MAX) +#if USHRT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif USHRT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif + #else if(sizeof(unsigned short) == 2) { msgpack_pack_real_uint16(x, d); @@ -482,14 +532,24 @@ if(sizeof(unsigned short) == 2) { msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d) { -#if defined(SIZEOF_INT) || defined(UINT_MAX) -#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU +#if defined(SIZEOF_INT) +#if SIZEOF_INT == 2 msgpack_pack_real_uint16(x, d); -#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU +#elif SIZEOF_INT == 4 msgpack_pack_real_uint32(x, d); #else msgpack_pack_real_uint64(x, d); #endif + +#elif defined(UINT_MAX) +#if UINT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif UINT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif + #else if(sizeof(unsigned int) == 2) { msgpack_pack_real_uint16(x, d); @@ -503,18 +563,28 @@ if(sizeof(unsigned int) == 2) { msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d) { -#if defined(SIZEOF_LONG) || defined(ULONG_MAX) -#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL +#if defined(SIZEOF_LONG) +#if SIZEOF_LONG == 2 msgpack_pack_real_uint16(x, d); -#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL +#elif SIZEOF_LONG == 4 msgpack_pack_real_uint32(x, d); #else msgpack_pack_real_uint64(x, d); #endif -#else -if(sizeof(unsigned int) == 2) { + +#elif defined(ULONG_MAX) +#if ULONG_MAX == 0xffffUL msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned int) == 4) { +#elif ULONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif + +#else +if(sizeof(unsigned long) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned long) == 4) { msgpack_pack_real_uint32(x, d); } else { msgpack_pack_real_uint64(x, d); @@ -524,14 +594,24 @@ if(sizeof(unsigned int) == 2) { msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d) { -#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX) -#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL +#if defined(SIZEOF_LONG_LONG) +#if SIZEOF_LONG_LONG == 2 msgpack_pack_real_uint16(x, d); -#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL +#elif SIZEOF_LONG_LONG == 4 msgpack_pack_real_uint32(x, d); #else msgpack_pack_real_uint64(x, d); #endif + +#elif defined(ULLONG_MAX) +#if ULLONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif ULLONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif + #else if(sizeof(unsigned long long) == 2) { msgpack_pack_real_uint16(x, d); @@ -554,19 +634,24 @@ if(sizeof(unsigned long long) == 2) { msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { - union { char buf[4]; uint32_t num; } f; + union { float f; uint32_t i; } mem; + mem.f = d; unsigned char buf[5]; - *((float*)&f.buf) = d; // FIXME - buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(f.num); + buf[0] = 0xca; _msgpack_store32(&buf[1], mem.i); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { - union { char buf[8]; uint64_t num; } f; + union { double f; uint64_t i; } mem; + mem.f = d; unsigned char buf[9]; - *((double*)&f.buf) = d; // FIXME - buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(f.num); + buf[0] = 0xcb; +#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi + // https://github.com/msgpack/msgpack-perl/pull/1 + mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL); +#endif + _msgpack_store64(&buf[1], mem.i); msgpack_pack_append_buffer(x, buf, 9); } @@ -610,11 +695,11 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { unsigned char buf[3]; - buf[0] = 0xdc; *(uint16_t*)&buf[1] = _msgpack_be16(n); + buf[0] = 0xdc; _msgpack_store16(&buf[1], (uint16_t)n); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdd; *(uint32_t*)&buf[1] = _msgpack_be32(n); + buf[0] = 0xdd; _msgpack_store32(&buf[1], (uint32_t)n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -631,11 +716,11 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(n < 65536) { unsigned char buf[3]; - buf[0] = 0xde; *(uint16_t*)&buf[1] = _msgpack_be16(n); + buf[0] = 0xde; _msgpack_store16(&buf[1], (uint16_t)n); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdf; *(uint32_t*)&buf[1] = _msgpack_be32(n); + buf[0] = 0xdf; _msgpack_store32(&buf[1], (uint32_t)n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -648,15 +733,15 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) { if(l < 32) { - unsigned char d = 0xa0 | l; + unsigned char d = 0xa0 | (uint8_t)l; msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(l < 65536) { unsigned char buf[3]; - buf[0] = 0xda; *(uint16_t*)&buf[1] = _msgpack_be16(l); + buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdb; *(uint32_t*)&buf[1] = _msgpack_be32(l); + buf[0] = 0xdb; _msgpack_store32(&buf[1], (uint32_t)l); msgpack_pack_append_buffer(x, buf, 5); } } diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h index 106158e..4fedbd8 100644 --- a/msgpack/sysdep.h +++ b/msgpack/sysdep.h @@ -1,7 +1,7 @@ /* * MessagePack system dependencies * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,8 +18,9 @@ #ifndef MSGPACK_SYSDEP_H__ #define MSGPACK_SYSDEP_H__ - -#ifdef _MSC_VER +#include +#include +#if defined(_MSC_VER) && _MSC_VER < 1600 typedef __int8 int8_t; typedef unsigned __int8 uint8_t; typedef __int16 int16_t; @@ -28,26 +29,38 @@ typedef __int32 int32_t; typedef unsigned __int32 uint32_t; typedef __int64 int64_t; typedef unsigned __int64 uint64_t; +#elif defined(_MSC_VER) // && _MSC_VER >= 1600 +#include #else -#include #include #include #endif - #ifdef _WIN32 +#define _msgpack_atomic_counter_header typedef long _msgpack_atomic_counter_t; #define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr) #define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr) +#elif defined(__GNUC__) && ((__GNUC__*10 + __GNUC_MINOR__) < 41) +#define _msgpack_atomic_counter_header "gcc_atomic.h" #else typedef unsigned int _msgpack_atomic_counter_t; #define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1) #define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1) #endif - #ifdef _WIN32 -#include + +#ifdef __cplusplus +/* numeric_limits::min,max */ +#ifdef max +#undef max +#endif +#ifdef min +#undef min +#endif +#endif + #else #include /* __BYTE_ORDER */ #endif @@ -57,15 +70,45 @@ typedef unsigned int _msgpack_atomic_counter_t; #define __LITTLE_ENDIAN__ #elif __BYTE_ORDER == __BIG_ENDIAN #define __BIG_ENDIAN__ +#elif _WIN32 +#define __LITTLE_ENDIAN__ #endif #endif + #ifdef __LITTLE_ENDIAN__ -#define _msgpack_be16(x) ntohs(x) -#define _msgpack_be32(x) ntohl(x) +#ifdef _WIN32 +# if defined(ntohs) +# define _msgpack_be16(x) ntohs(x) +# elif defined(_byteswap_ushort) || (defined(_MSC_VER) && _MSC_VER >= 1400) +# define _msgpack_be16(x) ((uint16_t)_byteswap_ushort((unsigned short)x)) +# else +# define _msgpack_be16(x) ( \ + ((((uint16_t)x) << 8) ) | \ + ((((uint16_t)x) >> 8) ) ) +# endif +#else +# define _msgpack_be16(x) ntohs(x) +#endif -#if defined(_byteswap_uint64) +#ifdef _WIN32 +# if defined(ntohl) +# define _msgpack_be32(x) ntohl(x) +# elif defined(_byteswap_ulong) || (defined(_MSC_VER) && _MSC_VER >= 1400) +# define _msgpack_be32(x) ((uint32_t)_byteswap_ulong((unsigned long)x)) +# else +# define _msgpack_be32(x) \ + ( ((((uint32_t)x) << 24) ) | \ + ((((uint32_t)x) << 8) & 0x00ff0000U ) | \ + ((((uint32_t)x) >> 8) & 0x0000ff00U ) | \ + ((((uint32_t)x) >> 24) ) ) +# endif +#else +# define _msgpack_be32(x) ntohl(x) +#endif + +#if defined(_byteswap_uint64) || (defined(_MSC_VER) && _MSC_VER >= 1400) # define _msgpack_be64(x) (_byteswap_uint64(x)) #elif defined(bswap_64) # define _msgpack_be64(x) bswap_64(x) @@ -73,22 +116,80 @@ typedef unsigned int _msgpack_atomic_counter_t; # define _msgpack_be64(x) __DARWIN_OSSwapInt64(x) #else #define _msgpack_be64(x) \ - ( ((((uint64_t)x) << 56) & 0xff00000000000000ULL ) | \ - ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \ - ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \ - ((((uint64_t)x) << 8) & 0x000000ff00000000ULL ) | \ - ((((uint64_t)x) >> 8) & 0x00000000ff000000ULL ) | \ - ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \ - ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \ - ((((uint64_t)x) >> 56) & 0x00000000000000ffULL ) ) + ( ((((uint64_t)x) << 56) ) | \ + ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \ + ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \ + ((((uint64_t)x) << 8) & 0x000000ff00000000ULL ) | \ + ((((uint64_t)x) >> 8) & 0x00000000ff000000ULL ) | \ + ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \ + ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \ + ((((uint64_t)x) >> 56) ) ) #endif +#define _msgpack_load16(cast, from) ((cast)( \ + (((uint16_t)((uint8_t*)(from))[0]) << 8) | \ + (((uint16_t)((uint8_t*)(from))[1]) ) )) + +#define _msgpack_load32(cast, from) ((cast)( \ + (((uint32_t)((uint8_t*)(from))[0]) << 24) | \ + (((uint32_t)((uint8_t*)(from))[1]) << 16) | \ + (((uint32_t)((uint8_t*)(from))[2]) << 8) | \ + (((uint32_t)((uint8_t*)(from))[3]) ) )) + +#define _msgpack_load64(cast, from) ((cast)( \ + (((uint64_t)((uint8_t*)(from))[0]) << 56) | \ + (((uint64_t)((uint8_t*)(from))[1]) << 48) | \ + (((uint64_t)((uint8_t*)(from))[2]) << 40) | \ + (((uint64_t)((uint8_t*)(from))[3]) << 32) | \ + (((uint64_t)((uint8_t*)(from))[4]) << 24) | \ + (((uint64_t)((uint8_t*)(from))[5]) << 16) | \ + (((uint64_t)((uint8_t*)(from))[6]) << 8) | \ + (((uint64_t)((uint8_t*)(from))[7]) ) )) + #else + #define _msgpack_be16(x) (x) #define _msgpack_be32(x) (x) #define _msgpack_be64(x) (x) + +#define _msgpack_load16(cast, from) ((cast)( \ + (((uint16_t)((uint8_t*)from)[0]) << 8) | \ + (((uint16_t)((uint8_t*)from)[1]) ) )) + +#define _msgpack_load32(cast, from) ((cast)( \ + (((uint32_t)((uint8_t*)from)[0]) << 24) | \ + (((uint32_t)((uint8_t*)from)[1]) << 16) | \ + (((uint32_t)((uint8_t*)from)[2]) << 8) | \ + (((uint32_t)((uint8_t*)from)[3]) ) )) + +#define _msgpack_load64(cast, from) ((cast)( \ + (((uint64_t)((uint8_t*)from)[0]) << 56) | \ + (((uint64_t)((uint8_t*)from)[1]) << 48) | \ + (((uint64_t)((uint8_t*)from)[2]) << 40) | \ + (((uint64_t)((uint8_t*)from)[3]) << 32) | \ + (((uint64_t)((uint8_t*)from)[4]) << 24) | \ + (((uint64_t)((uint8_t*)from)[5]) << 16) | \ + (((uint64_t)((uint8_t*)from)[6]) << 8) | \ + (((uint64_t)((uint8_t*)from)[7]) ) )) #endif +#define _msgpack_store16(to, num) \ + do { uint16_t val = _msgpack_be16(num); memcpy(to, &val, 2); } while(0) +#define _msgpack_store32(to, num) \ + do { uint32_t val = _msgpack_be32(num); memcpy(to, &val, 4); } while(0) +#define _msgpack_store64(to, num) \ + do { uint64_t val = _msgpack_be64(num); memcpy(to, &val, 8); } while(0) + +/* +#define _msgpack_load16(cast, from) \ + ({ cast val; memcpy(&val, (char*)from, 2); _msgpack_be16(val); }) +#define _msgpack_load32(cast, from) \ + ({ cast val; memcpy(&val, (char*)from, 4); _msgpack_be32(val); }) +#define _msgpack_load64(cast, from) \ + ({ cast val; memcpy(&val, (char*)from, 8); _msgpack_be64(val); }) +*/ + + #endif /* msgpack/sysdep.h */ diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 63d90a8..959d351 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -1,7 +1,7 @@ /* * MessagePack unpacking routine template * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,8 @@ #ifndef MSGPACK_UNPACK_DEFINE_H__ #define MSGPACK_UNPACK_DEFINE_H__ -#include "sysdep.h" +#include "msgpack/sysdep.h" +#include #include #include #include @@ -28,8 +29,8 @@ extern "C" { #endif -#ifndef MSGPACK_MAX_STACK_SIZE -#define MSGPACK_MAX_STACK_SIZE 16 +#ifndef MSGPACK_EMBED_STACK_SIZE +#define MSGPACK_EMBED_STACK_SIZE 32 #endif diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 42fe3a1..711b163 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -1,7 +1,7 @@ /* * MessagePack unpacking routine template * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -50,11 +50,7 @@ msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; size_t count; unsigned int ct; - - union { - size_t curr; - msgpack_unpack_object map_key; - }; + msgpack_unpack_object map_key; }; msgpack_unpack_struct_decl(_context) { @@ -62,7 +58,12 @@ msgpack_unpack_struct_decl(_context) { unsigned int cs; unsigned int trail; unsigned int top; - msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE]; + /* + msgpack_unpack_struct(_stack)* stack; + unsigned int stack_size; + msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE]; + */ + msgpack_unpack_struct(_stack) stack[MSGPACK_EMBED_STACK_SIZE]; }; @@ -71,9 +72,22 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; + /* + ctx->stack = ctx->embed_stack; + ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; + */ ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); } +/* +msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx) +{ + if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { + free(ctx->stack); + } +} +*/ + msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) { return (ctx)->stack[0].obj; @@ -82,6 +96,8 @@ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) { + assert(len >= *off); + const unsigned char* p = (unsigned char*)data + *off; const unsigned char* const pe = (unsigned char*)data + len; const void* n = NULL; @@ -89,8 +105,10 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c unsigned int trail = ctx->trail; unsigned int cs = ctx->cs; unsigned int top = ctx->top; - msgpack_unpack_struct(_stack)* stack = ctx->stack; + /* + unsigned int stack_size = ctx->stack_size; + */ msgpack_unpack_user* user = &ctx->user; msgpack_unpack_object obj; @@ -98,8 +116,6 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c int ret; - assert(len >= *off); - #define push_simple_value(func) \ if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \ goto _push @@ -122,25 +138,38 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c goto _fixed_trail_again #define start_container(func, count_, ct_) \ + if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \ if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ if((count_) == 0) { obj = stack[top].obj; goto _push; } \ - if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ stack[top].ct = ct_; \ - stack[top].curr = 0; \ stack[top].count = count_; \ + ++top; \ /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ /*printf("stack push %d\n", top);*/ \ - ++top; \ + /* FIXME \ + if(top >= stack_size) { \ + if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \ + size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \ + size_t nsize = csize * 2; \ + msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \ + if(tmp == NULL) { goto _failed; } \ + memcpy(tmp, ctx->stack, csize); \ + ctx->stack = stack = tmp; \ + ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ + } else { \ + size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \ + msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \ + if(tmp == NULL) { goto _failed; } \ + ctx->stack = stack = tmp; \ + ctx->stack_size = stack_size = stack_size * 2; \ + } \ + } \ + */ \ goto _header_again #define NEXT_CS(p) \ ((unsigned int)*p & 0x1f) -#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) _msgpack_be16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) _msgpack_be32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) _msgpack_be64(*(uint64_t*)ptr) - #ifdef USE_CASE_RANGE #define SWITCH_RANGE_BEGIN switch(*p) { #define SWITCH_RANGE(FROM, TO) case FROM ... TO: @@ -228,70 +257,74 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c //case CS_ //case CS_ case CS_FLOAT: { - union { uint32_t num; char buf[4]; } f; - f.num = PTR_CAST_32(n); // FIXME - push_fixed_value(_float, *((float*)f.buf)); } + union { uint32_t i; float f; } mem; + mem.i = _msgpack_load32(uint32_t,n); + push_fixed_value(_float, mem.f); } case CS_DOUBLE: { - union { uint64_t num; char buf[8]; } f; - f.num = PTR_CAST_64(n); // FIXME - push_fixed_value(_double, *((double*)f.buf)); } + union { uint64_t i; double f; } mem; + mem.i = _msgpack_load64(uint64_t,n); +#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi + // https://github.com/msgpack/msgpack-perl/pull/1 + mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL); +#endif + push_fixed_value(_double, mem.f); } case CS_UINT_8: - push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n)); + push_fixed_value(_uint8, *(uint8_t*)n); case CS_UINT_16: - push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n)); + push_fixed_value(_uint16, _msgpack_load16(uint16_t,n)); case CS_UINT_32: - push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n)); + push_fixed_value(_uint32, _msgpack_load32(uint32_t,n)); case CS_UINT_64: - push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n)); + push_fixed_value(_uint64, _msgpack_load64(uint64_t,n)); case CS_INT_8: - push_fixed_value(_int8, (int8_t)PTR_CAST_8(n)); + push_fixed_value(_int8, *(int8_t*)n); case CS_INT_16: - push_fixed_value(_int16, (int16_t)PTR_CAST_16(n)); + push_fixed_value(_int16, _msgpack_load16(int16_t,n)); case CS_INT_32: - push_fixed_value(_int32, (int32_t)PTR_CAST_32(n)); + push_fixed_value(_int32, _msgpack_load32(int32_t,n)); case CS_INT_64: - push_fixed_value(_int64, (int64_t)PTR_CAST_64(n)); + push_fixed_value(_int64, _msgpack_load64(int64_t,n)); //case CS_ //case CS_ //case CS_BIG_INT_16: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero); //case CS_BIG_INT_32: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero); //case ACS_BIG_INT_VALUE: //_big_int_zero: // // FIXME // push_variable_value(_big_int, data, n, trail); //case CS_BIG_FLOAT_16: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero); //case CS_BIG_FLOAT_32: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero); //case ACS_BIG_FLOAT_VALUE: //_big_float_zero: // // FIXME // push_variable_value(_big_float, data, n, trail); case CS_RAW_16: - again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); + again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero); case CS_RAW_32: - again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); + again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero); case ACS_RAW_VALUE: _raw_zero: push_variable_value(_raw, data, n, trail); case CS_ARRAY_16: - start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM); case CS_ARRAY_32: /* FIXME security guard */ - start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM); case CS_MAP_16: - start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY); case CS_MAP_32: /* FIXME security guard */ - start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); default: goto _failed; @@ -303,9 +336,8 @@ _push: c = &stack[top-1]; switch(c->ct) { case CT_ARRAY_ITEM: - if(msgpack_unpack_callback(_array_item)(user, c->curr, &c->obj, obj) < 0) { goto _failed; } - if(++c->curr == c->count) { - msgpack_unpack_callback(_array_end)(user, &c->obj); + if(msgpack_unpack_callback(_array_item)(user, &c->obj, obj) < 0) { goto _failed; } + if(--c->count == 0) { obj = c->obj; --top; /*printf("stack pop %d\n", top);*/ @@ -319,7 +351,6 @@ _push: case CT_MAP_VALUE: if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } if(--c->count == 0) { - msgpack_unpack_callback(_map_end)(user, &c->obj); obj = c->obj; --top; /*printf("stack pop %d\n", top);*/ @@ -379,8 +410,4 @@ _end: #undef start_container #undef NEXT_CS -#undef PTR_CAST_8 -#undef PTR_CAST_16 -#undef PTR_CAST_32 -#undef PTR_CAST_64 From 188da01777d83e4024c3c7799892a492e4ec25e8 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 26 Jun 2012 15:19:59 +0900 Subject: [PATCH 0525/1172] Fix new version of msgpack. --- msgpack/unpack.h | 2 +- msgpack/unpack_template.h | 17 ++++++++++++----- setup.py | 1 + 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 2659a97..a106f9c 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -16,7 +16,7 @@ * limitations under the License. */ -#define MSGPACK_MAX_STACK_SIZE (1024) +#define MSGPACK_EMBED_STACK_SIZE (1024) #include "unpack_define.h" typedef struct unpack_user { diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 711b163..b844a24 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -48,6 +48,7 @@ msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; + size_t size; size_t count; unsigned int ct; msgpack_unpack_object map_key; @@ -140,9 +141,12 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c #define start_container(func, count_, ct_) \ if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \ if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ - if((count_) == 0) { obj = stack[top].obj; goto _push; } \ + if((count_) == 0) { obj = stack[top].obj; \ + msgpack_unpack_callback(func##_end)(user, &obj); \ + goto _push; } \ stack[top].ct = ct_; \ - stack[top].count = count_; \ + stack[top].size = count_; \ + stack[top].count = 0; \ ++top; \ /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ /*printf("stack push %d\n", top);*/ \ @@ -336,9 +340,10 @@ _push: c = &stack[top-1]; switch(c->ct) { case CT_ARRAY_ITEM: - if(msgpack_unpack_callback(_array_item)(user, &c->obj, obj) < 0) { goto _failed; } - if(--c->count == 0) { + if(msgpack_unpack_callback(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } + if(++c->count == c->size) { obj = c->obj; + msgpack_unpack_callback(_array_end)(user, &obj); --top; /*printf("stack pop %d\n", top);*/ goto _push; @@ -350,8 +355,9 @@ _push: goto _header_again; case CT_MAP_VALUE: if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } - if(--c->count == 0) { + if(++c->count == c->size) { obj = c->obj; + msgpack_unpack_callback(_map_end)(user, &obj); --top; /*printf("stack pop %d\n", top);*/ goto _push; @@ -411,3 +417,4 @@ _end: #undef NEXT_CS +/* vim: set ts=4 sw=4 noexpandtab */ diff --git a/setup.py b/setup.py index 115d3b8..7e78b4c 100755 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ if sys.platform == 'win32': msgpack_mod = Extension('msgpack._msgpack', sources=sources, libraries=libraries, + include_dirs=['.'], ) del sources, libraries From 58eb7d0ce857b0ffb5e0ee4addf5ad1b3b47afa9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 26 Jun 2012 15:34:51 +0900 Subject: [PATCH 0526/1172] Update changelog --- ChangeLog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 3a9277c..2fc9c18 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -5,6 +5,7 @@ Changes ------- * Drop supporting Python 2.5 and unify tests for Py2 and Py3. +* Use new version of msgpack-c. It packs correctly on big endian platforms. Bugs fixed ---------- From ebe4c1f4bc31bc1f47831d72819cac5a2a584604 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 26 Jun 2012 17:27:29 +0900 Subject: [PATCH 0527/1172] manage to compile on windows Use C++ compiler to build. --- msgpack/_msgpack.pyx | 4 ++-- msgpack/pack.h | 2 +- setup.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) mode change 100755 => 100644 setup.py diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 990f585..526c003 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -222,8 +222,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint template_init(&ctx) ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL - ctx.user.encoding = enc - ctx.user.unicode_errors = err + ctx.user.encoding = enc + ctx.user.unicode_errors = err if object_hook is not None: if not PyCallable_Check(object_hook): raise TypeError("object_hook must be a callable.") diff --git a/msgpack/pack.h b/msgpack/pack.h index d36b436..4c0373e 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -77,7 +77,7 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ if (len + l > bs) { bs = (len + l) * 2; - buf = realloc(buf, bs); + buf = (char*)realloc(buf, bs); if (!buf) return -1; } memcpy(buf + len, data, l); diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index 7e78b4c..149b282 --- a/setup.py +++ b/setup.py @@ -53,6 +53,7 @@ msgpack_mod = Extension('msgpack._msgpack', sources=sources, libraries=libraries, include_dirs=['.'], + language='c++', ) del sources, libraries From 06ed24a529eba164b23089a7653427f62dfb32e2 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 26 Jun 2012 17:37:22 +0900 Subject: [PATCH 0528/1172] Fix setup scripts. Support _msgpack.cpp --- MANIFEST.in | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index bf312d5..4e85759 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include setup.py include COPYING -recursive-include msgpack *.h *.c *.pyx +recursive-include msgpack *.h *.c *.pyx *.cpp recursive-include test *.py recursive-include test3 *.py diff --git a/setup.py b/setup.py index 149b282..3f8f9f8 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ if have_cython: cython_compiler.default_options) sdist.__init__(self, *args, **kwargs) else: - sources = ['msgpack/_msgpack.c'] + sources = ['msgpack/_msgpack.cpp'] for f in sources: if not os.path.exists(f): From 288e82029353e3c3256341c0e32d16ca32d33700 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 27 Jun 2012 18:05:35 +0900 Subject: [PATCH 0529/1172] prepare 0.2 --- MANIFEST.in | 2 +- README.rst | 78 +++++++++++++++++++++++++++++++++++++++++++---------- setup.py | 13 ++++----- 3 files changed, 70 insertions(+), 23 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 4e85759..e1912ca 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include setup.py include COPYING +include README.rst recursive-include msgpack *.h *.c *.pyx *.cpp recursive-include test *.py -recursive-include test3 *.py diff --git a/README.rst b/README.rst index 994492e..f392b67 100644 --- a/README.rst +++ b/README.rst @@ -3,33 +3,83 @@ MessagePack Python Binding =========================== :author: INADA Naoki -:version: 0.1.0 -:date: 2009-07-12 +:version: 0.2.0 +:date: 2012-06-27 HOW TO USE ----------- -You can read document in docstring after `import msgpack` + +one-shot pack & unpack +^^^^^^^^^^^^^^^^^^^^^^ + +Use ``packb`` for packing and ``unpackb`` for unpacking. +msgpack provides ``dumps`` and ``loads`` as alias for compatibility with +``json`` and ``pickle``. + +``pack`` and ``dump`` packs to file-like object. +``unpack`` and ``load`` unpacks from file-like object. + + >>> import msgpack + >>> msgpack.packb([1, 2, 3]) + '\x93\x01\x02\x03' + >>> msgpack.unpackb(_) + (1, 2, 3) + + +``unpack`` unpacks msgpack's array to Python's tuple. +To unpack it to list, Use ``use_list`` option. + + >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=True) + [1, 2, 3] + +Read docstring for other options. + + +streaming unpacking +^^^^^^^^^^^^^^^^^^^ + +``Unpacker`` is "streaming unpacker". It unpacks multiple objects from one +stream. + +:: + + import msgpack + from io import BytesIO + + buf = BytesIO() + for i in range(100): + buf.write(msgpack.packb(range(i))) + + buf.seek(0) + + unpacker = msgpack.Unpacker() + while True: + data = buf.read(4) + if not data: + break + unpacker.seed(buf.read(16)) + for unpacked in unpacker: + print unpacked INSTALL --------- -Cython_ is required to build msgpack. +You can use ``pip`` or ``easy_install`` to install msgpack:: -.. _Cython: http://www.cython.org/ + $ easy_install msgpack-python + or + $ pip install msgpack-python -posix -'''''' -You can install msgpack in common way. - - $ python setup.py install Windows '''''''' -MessagePack requires gcc currently. So you need to prepare -MinGW GCC. +msgpack provides some binary distribution for Windows. +You can install msgpack without compiler with them. + +When you can't use binary distribution, you need to install Visual Studio +or Windows SDK on Windows. (NOTE: Visual C++ Express 2010 doesn't support +amd64. Windows SDK is recommanded way to build amd64 msgpack without any fee.) - $ python setup.py build -c mingw32 - $ python setup.py install TEST ---- diff --git a/setup.py b/setup.py index 3f8f9f8..5802c3c 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 1, 14, 'dev') +version = (0, 2, 0, 'dev') import os import sys @@ -59,13 +59,10 @@ del sources, libraries desc = 'MessagePack (de)serializer.' -long_desc = """MessagePack (de)serializer for Python. - -What's MessagePack? (from http://msgpack.org/) - - MessagePack is a binary-based efficient data interchange format that is - focused on high performance. It is like JSON, but very fast and small. -""" +f = open('README.rst') +long_desc = f.read() +f.close() +del f setup(name='msgpack-python', author='INADA Naoki', From dd5b1e265a24a6ddc6ffe681d9d632a8240ce2ad Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 27 Jun 2012 18:07:02 +0900 Subject: [PATCH 0530/1172] remove deprecated api. --- ChangeLog.rst | 5 +++-- msgpack/__init__.py | 9 --------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 2fc9c18..e255e19 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,11 +1,12 @@ -0.1.14 +0.2.0 ======= -:release date: NOT RELEASED YET +:release date: 2012-06-27 Changes ------- * Drop supporting Python 2.5 and unify tests for Py2 and Py3. * Use new version of msgpack-c. It packs correctly on big endian platforms. +* Remove deprecated packs and unpacks API. Bugs fixed ---------- diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 6b9735e..cdf045f 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -9,12 +9,3 @@ loads = unpackb dump = pack dumps = packb -def packs(*args, **kw): - from warnings import warn - warn("msgpack.packs() is deprecated. Use packb() instead.", DeprecationWarning) - return packb(*args, **kw) - -def unpacks(*args, **kw): - from warnings import warn - warn("msgpack.unpacks() is deprecated. Use unpackb() instead.", DeprecationWarning) - return unpackb(*args, **kw) From 52a7c02a50309fba0652279fdb4065821cf4d69c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 27 Jun 2012 18:15:44 +0900 Subject: [PATCH 0531/1172] update .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 9452edd..7918394 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,8 @@ dist/* *.pyc *.pyo *.so +*~ msgpack/__version__.py msgpack/_msgpack.c +msgpack/_msgpack.cpp +*.egg-info From 002a941b432d62e4a24f17309c8c09cbf394cb56 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 27 Jun 2012 18:16:59 +0900 Subject: [PATCH 0532/1172] Use setuptools to build egg package. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5802c3c..70d9b06 100644 --- a/setup.py +++ b/setup.py @@ -5,8 +5,8 @@ version = (0, 2, 0, 'dev') import os import sys from glob import glob -from distutils.core import setup, Extension from distutils.command.sdist import sdist +from setuptools import setup, Extension try: from Cython.Distutils import build_ext From 8514871c9b5b3063ed8042bbd8043522696cb5b1 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 27 Jun 2012 18:23:47 +0900 Subject: [PATCH 0533/1172] release 0.2.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 70d9b06..43f0d3b 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 2, 0, 'dev') +version = (0, 2, 0) import os import sys From 4bff55db9fe73e23b6da63a8b34c3679754549de Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 27 Jun 2012 18:25:56 +0900 Subject: [PATCH 0534/1172] Fix rst syntax. --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index f392b67..1da6ece 100644 --- a/README.rst +++ b/README.rst @@ -72,7 +72,7 @@ You can use ``pip`` or ``easy_install`` to install msgpack:: Windows -'''''''' +^^^^^^^ msgpack provides some binary distribution for Windows. You can install msgpack without compiler with them. From 2122b46b8489f3cf414f5ebb137f76ecc9fd9598 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 4 Jul 2012 14:58:36 +0900 Subject: [PATCH 0535/1172] Fix using deprecated api in tests. --- test/test_case.py | 12 ++++++------ test/test_except.py | 4 ++-- test/test_format.py | 4 ++-- test/test_obj.py | 18 +++++++++--------- test/test_pack.py | 20 ++++++++++---------- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/test/test_case.py b/test/test_case.py index 2f42316..b88714d 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -3,13 +3,13 @@ from nose import main from nose.tools import * -from msgpack import packs, unpacks +from msgpack import packb, unpackb def check(length, obj): - v = packs(obj) + v = packb(obj) assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) - assert_equal(unpacks(v), obj) + assert_equal(unpackb(v), obj) def test_1(): for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1, @@ -70,8 +70,8 @@ def test_array32(): def match(obj, buf): - assert_equal(packs(obj), buf) - assert_equal(unpacks(buf), obj) + assert_equal(packb(obj), buf) + assert_equal(unpackb(buf), obj) def test_match(): cases = [ @@ -99,7 +99,7 @@ def test_match(): match(v, p) def test_unicode(): - assert_equal(b'foobar', unpacks(packs('foobar'))) + assert_equal(b'foobar', unpackb(packb('foobar'))) if __name__ == '__main__': main() diff --git a/test/test_except.py b/test/test_except.py index 574728f..ad02cb6 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -2,12 +2,12 @@ # coding: utf-8 from nose.tools import * -from msgpack import packs, unpacks +from msgpack import packb, unpackb import datetime def test_raise_on_find_unsupported_value(): - assert_raises(TypeError, packs, datetime.datetime.now()) + assert_raises(TypeError, packb, datetime.datetime.now()) if __name__ == '__main__': from nose import main diff --git a/test/test_format.py b/test/test_format.py index 022e680..c03b3e2 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -3,10 +3,10 @@ from nose import main from nose.tools import * -from msgpack import unpacks +from msgpack import unpackb def check(src, should): - assert_equal(unpacks(src), should) + assert_equal(unpackb(src), should) def testSimpleValue(): check(b"\x93\xc0\xc2\xc3", diff --git a/test/test_obj.py b/test/test_obj.py index 6357cfc..d155b73 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -4,7 +4,7 @@ from nose import main from nose.tools import * -from msgpack import packs, unpacks +from msgpack import packb, unpackb def _decode_complex(obj): if b'__complex__' in obj: @@ -17,26 +17,26 @@ def _encode_complex(obj): return obj def test_encode_hook(): - packed = packs([3, 1+2j], default=_encode_complex) - unpacked = unpacks(packed) + packed = packb([3, 1+2j], default=_encode_complex) + unpacked = unpackb(packed) eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2}) def test_decode_hook(): - packed = packs([3, {b'__complex__': True, b'real': 1, b'imag': 2}]) - unpacked = unpacks(packed, object_hook=_decode_complex) + packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}]) + unpacked = unpackb(packed, object_hook=_decode_complex) eq_(unpacked[1], 1+2j) @raises(ValueError) def test_bad_hook(): - packed = packs([3, 1+2j], default=lambda o: o) - unpacked = unpacks(packed) + packed = packb([3, 1+2j], default=lambda o: o) + unpacked = unpackb(packed) def _arr_to_str(arr): return ''.join(str(c) for c in arr) def test_array_hook(): - packed = packs([1,2,3]) - unpacked = unpacks(packed, list_hook=_arr_to_str) + packed = packb([1,2,3]) + unpacked = unpackb(packed, list_hook=_arr_to_str) eq_(unpacked, '123') if __name__ == '__main__': diff --git a/test/test_pack.py b/test/test_pack.py index 88b6dac..898cdb9 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -6,12 +6,12 @@ from nose import main from nose.tools import * from nose.plugins.skip import SkipTest -from msgpack import packs, unpacks, Unpacker, Packer +from msgpack import packb, unpackb, Unpacker, Packer from io import BytesIO def check(data): - re = unpacks(packs(data)) + re = unpackb(packb(data)) assert_equal(re, data) def testPack(): @@ -33,7 +33,7 @@ def testPackUnicode(): six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("РуÑÑкий текÑÑ‚"), ] for td in test_data: - re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8') + re = unpackb(packb(td, encoding='utf-8'), encoding='utf-8') assert_equal(re, td) packer = Packer(encoding='utf-8') data = packer.pack(td) @@ -49,7 +49,7 @@ def testPackUTF32(): six.u("РуÑÑкий текÑÑ‚"), ] for td in test_data: - re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32') + re = unpackb(packb(td, encoding='utf-32'), encoding='utf-32') assert_equal(re, td) except LookupError: raise SkipTest @@ -62,28 +62,28 @@ def testPackBytes(): check(td) def testIgnoreUnicodeErrors(): - re = unpacks(packs(b'abc\xeddef'), + re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore') assert_equal(re, "abcdef") @raises(UnicodeDecodeError) def testStrictUnicodeUnpack(): - unpacks(packs(b'abc\xeddef'), encoding='utf-8') + unpackb(packb(b'abc\xeddef'), encoding='utf-8') @raises(UnicodeEncodeError) def testStrictUnicodePack(): - packs(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict') + packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict') def testIgnoreErrorsPack(): - re = unpacks(packs(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8') + re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8') assert_equal(re, six.u("abcdef")) @raises(TypeError) def testNoEncoding(): - packs(six.u("abc"), encoding=None) + packb(six.u("abc"), encoding=None) def testDecodeBinary(): - re = unpacks(packs("abc"), encoding=None) + re = unpackb(packb("abc"), encoding=None) assert_equal(re, b"abc") if __name__ == '__main__': From e133c7fd27a515feba7691d6ad37cc9d58dc5b42 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 13 Jul 2012 21:15:11 +0900 Subject: [PATCH 0536/1172] Start 0.2.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 43f0d3b..b46fa91 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 2, 0) +version = (0, 2, 1, 'dev1') import os import sys From 7b1167044b17572126fc69b89eb6fa9c0a5afb91 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 13 Jul 2012 21:28:16 +0900 Subject: [PATCH 0537/1172] Add max_buffer_size to Unpacker. --- ChangeLog.rst | 16 ++++++++++ msgpack/_msgpack.pyx | 71 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 73 insertions(+), 14 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index e255e19..e86e3bd 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,19 @@ +0.2.1 +======= +:release date: NOT RELEASED YET + +Changes +------- +* Add ``max_buffer_size`` parameter to Unpacker. It limits internal buffer size + and allows unpack data from untrusted source safely. + +* Unpacker's buffer reallocation algorithm is less greedy now. It cause perforamce + derease in rare case but memory efficient and don't allocate than ``max_buffer_size``. + +Bugs fixed +---------- + + 0.2.0 ======= :release date: 2012-06-27 diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 526c003..96abb42 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -10,6 +10,9 @@ cdef extern from "Python.h": from libc.stdlib cimport * from libc.string cimport * +from libc.limits cimport * + + import gc _gc_disable = gc.disable _gc_enable = gc.enable @@ -35,6 +38,11 @@ cdef extern from "pack.h": cdef int DEFAULT_RECURSE_LIMIT=511 + +class BufferFull(Exception): + pass + + cdef class Packer(object): """MessagePack Packer @@ -193,7 +201,9 @@ cdef extern from "unpack.h": object template_data(template_context* ctx) -def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"): +def unpackb(object packed, object object_hook=None, object list_hook=None, + bint use_list=0, encoding=None, unicode_errors="strict", + ): """ Unpack packed_bytes to object. Returns an unpacked object.""" cdef template_context ctx @@ -243,12 +253,16 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint return None -def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"): +def unpack(object stream, object object_hook=None, object list_hook=None, + bint use_list=0, encoding=None, unicode_errors="strict", + ): """ unpack an object from stream. """ return unpackb(stream.read(), use_list=use_list, - object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors) + object_hook=object_hook, list_hook=list_hook, + encoding=encoding, unicode_errors=unicode_errors, + ) cdef class Unpacker(object): """ @@ -259,7 +273,7 @@ cdef class Unpacker(object): When `Unpacker` initialized with `file_like`, unpacker reads serialized data from it and `.feed()` method is not usable. - `read_size` is used as `file_like.read(read_size)`. (default: 1M) + `read_size` is used as `file_like.read(read_size)`. (default: 1024**2) If `use_list` is true, msgpack list is deserialized to Python list. Otherwise, it is deserialized to Python tuple. (default: False) @@ -272,11 +286,24 @@ cdef class Unpacker(object): `unicode_errors` is used for decoding bytes. - example:: + `max_buffer_size` limits size of data waiting unpacked. 0 means unlimited + (default). + Raises `BufferFull` exception when it is insufficient. + You shoud set this parameter when unpacking data from untrasted source. + + example of streaming deserialize from file-like object:: + + unpacker = Unpacker(file_like) + for o in unpacker: + do_something(o) + + example of streaming deserialize from socket:: unpacker = Unpacker() while 1: - buf = astream.read() + buf = sock.recv(1024**2) + if not buf: + break unpacker.feed(buf) for o in unpacker: do_something(o) @@ -293,6 +320,7 @@ cdef class Unpacker(object): cdef object _berrors cdef char *encoding cdef char *unicode_errors + cdef size_t max_buffer_size def __cinit__(self): self.buf = NULL @@ -303,7 +331,7 @@ cdef class Unpacker(object): def __init__(self, file_like=None, Py_ssize_t read_size=1024*1024, bint use_list=0, object object_hook=None, object list_hook=None, - encoding=None, unicode_errors='strict'): + encoding=None, unicode_errors='strict', int max_buffer_size=0): self.use_list = use_list self.file_like = file_like if file_like: @@ -314,6 +342,10 @@ cdef class Unpacker(object): self.buf = malloc(read_size) if self.buf == NULL: raise MemoryError("Unable to allocate internal buffer.") + if max_buffer_size: + self.max_buffer_size = max_buffer_size + else: + self.max_buffer_size = INT_MAX self.buf_size = read_size self.buf_head = 0 self.buf_tail = 0 @@ -355,28 +387,36 @@ cdef class Unpacker(object): cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len): cdef: char* buf = self.buf + char* new_buf size_t head = self.buf_head size_t tail = self.buf_tail size_t buf_size = self.buf_size size_t new_size if tail + _buf_len > buf_size: - if ((tail - head) + _buf_len)*2 < buf_size: + if ((tail - head) + _buf_len) <= buf_size: # move to front. memmove(buf, buf + head, tail - head) tail -= head head = 0 else: # expand buffer. - new_size = tail + _buf_len - if new_size < buf_size*2: - new_size = buf_size*2 - buf = realloc(buf, new_size) - if buf == NULL: + new_size = (tail-head) + _buf_len + if new_size > self.max_buffer_size: + raise BufferFull + new_size = min(new_size*2, self.max_buffer_size) + new_buf = malloc(new_size) + if new_buf == NULL: # self.buf still holds old buffer and will be freed during # obj destruction raise MemoryError("Unable to enlarge internal buffer.") + memcpy(new_buf, buf + head, tail - head) + free(buf) + + buf = new_buf buf_size = new_size + tail -= head + head = 0 memcpy(buf + tail, (_buf), _buf_len) self.buf = buf @@ -387,7 +427,10 @@ cdef class Unpacker(object): # prepare self.buf from file_like cdef fill_buffer(self): if self.file_like is not None: - next_bytes = self.file_like_read(self.read_size) + next_bytes = self.file_like_read( + max(self.read_size, + self.max_buffer_size - (self.buf_tail - self.buf_head) + )) if next_bytes: self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes)) From e5462ff72f16517e5e9eb434efad084d874e3967 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 20 Jul 2012 02:02:37 +0900 Subject: [PATCH 0538/1172] Add test for max_buffer_size. --- test/test_sequnpack.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 774fe1b..b1b80b2 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # coding: utf-8 -from msgpack import Unpacker +from msgpack import Unpacker, BufferFull +import nose def test_foobar(): unpacker = Unpacker(read_size=3) @@ -27,6 +28,19 @@ def test_foobar(): k += 1 assert k == len(b'foobar') -if __name__ == '__main__': - test_foobar() +def test_maxbuffersize(): + nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3) + unpacker = Unpacker(read_size=3, max_buffer_size=3) + unpacker.feed(b'fo') + nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob') + unpacker.feed(b'o') + assert ord('f') == next(unpacker) + unpacker.feed(b'b') + assert ord('o') == next(unpacker) + assert ord('o') == next(unpacker) + assert ord('b') == next(unpacker) + + +if __name__ == '__main__': + nose.main() From 53ca2bb648d423462b970493b8ed63cdaf70afe4 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 20 Jul 2012 02:02:54 +0900 Subject: [PATCH 0539/1172] raise ValueError when read_size > max_buffer_size. --- msgpack/_msgpack.pyx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 96abb42..00d6d90 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -338,14 +338,15 @@ cdef class Unpacker(object): self.file_like_read = file_like.read if not PyCallable_Check(self.file_like_read): raise ValueError("`file_like.read` must be a callable.") + if not max_buffer_size: + max_buffer_size = INT_MAX + self.max_buffer_size = max_buffer_size + if read_size > max_buffer_size: + raise ValueError("read_size should be less or equal to max_buffer_size") self.read_size = read_size self.buf = malloc(read_size) if self.buf == NULL: raise MemoryError("Unable to allocate internal buffer.") - if max_buffer_size: - self.max_buffer_size = max_buffer_size - else: - self.max_buffer_size = INT_MAX self.buf_size = read_size self.buf_head = 0 self.buf_tail = 0 From 59c8b51e5b3df8322bc4d2f639bcafd6a25eecaf Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 20 Jul 2012 02:05:43 +0900 Subject: [PATCH 0540/1172] Default value of read_size is min(1024**2, max_buffer_size) --- msgpack/_msgpack.pyx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 00d6d90..ebf1592 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -267,13 +267,13 @@ def unpack(object stream, object object_hook=None, object list_hook=None, cdef class Unpacker(object): """ Streaming unpacker. - read_size is used like file_like.read(read_size) `file_like` is a file-like object having `.read(n)` method. When `Unpacker` initialized with `file_like`, unpacker reads serialized data from it and `.feed()` method is not usable. - `read_size` is used as `file_like.read(read_size)`. (default: 1024**2) + `read_size` is used as `file_like.read(read_size)`. + (default: min(1024**2, max_buffer_size)) If `use_list` is true, msgpack list is deserialized to Python list. Otherwise, it is deserialized to Python tuple. (default: False) @@ -329,7 +329,7 @@ cdef class Unpacker(object): free(self.buf) self.buf = NULL - def __init__(self, file_like=None, Py_ssize_t read_size=1024*1024, bint use_list=0, + def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, object object_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0): self.use_list = use_list @@ -340,6 +340,8 @@ cdef class Unpacker(object): raise ValueError("`file_like.read` must be a callable.") if not max_buffer_size: max_buffer_size = INT_MAX + if not read_size: + read_size = min(max_buffer_size, 1024**2) self.max_buffer_size = max_buffer_size if read_size > max_buffer_size: raise ValueError("read_size should be less or equal to max_buffer_size") From bf4124f592b4ba54c9ec09fbc8a4f898bc229967 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 25 Jul 2012 09:19:10 +0900 Subject: [PATCH 0541/1172] Fix setup.py sdist doesn't generates c++ source. --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b46fa91..2e6ec84 100644 --- a/setup.py +++ b/setup.py @@ -32,9 +32,10 @@ if have_cython: class Sdist(sdist): def __init__(self, *args, **kwargs): + cy_opt = cython_compiler.default_options.copy() + cy_opt['cplus'] = True for src in glob('msgpack/*.pyx'): - cython_compiler.compile(glob('msgpack/*.pyx'), - cython_compiler.default_options) + cython_compiler.compile(glob('msgpack/*.pyx'), cy_opt) sdist.__init__(self, *args, **kwargs) else: sources = ['msgpack/_msgpack.cpp'] From 7c03f322fa73901620b2cd188186003dd9e02841 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 19 Aug 2012 01:05:41 +0900 Subject: [PATCH 0542/1172] Fix on SPARC Solaris. Use C++ only on Windows. Define ENDIAN macros from `sys.byteorder`. --- setup.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 2e6ec84..7d8a5d4 100644 --- a/setup.py +++ b/setup.py @@ -33,12 +33,12 @@ if have_cython: class Sdist(sdist): def __init__(self, *args, **kwargs): cy_opt = cython_compiler.default_options.copy() - cy_opt['cplus'] = True + #cy_opt['cplus'] = True for src in glob('msgpack/*.pyx'): cython_compiler.compile(glob('msgpack/*.pyx'), cy_opt) sdist.__init__(self, *args, **kwargs) else: - sources = ['msgpack/_msgpack.cpp'] + sources = ['msgpack/_msgpack.c'] for f in sources: if not os.path.exists(f): @@ -47,16 +47,24 @@ else: Sdist = sdist libraries = [] +language = 'c' if sys.platform == 'win32': libraries.append('ws2_32') + language = 'c++' + +if sys.byteorder == 'big': + macros = [('__BIG_ENDIAN__', '1')] +else: + macros = [('__LITTLE_ENDIAN__', '1')] msgpack_mod = Extension('msgpack._msgpack', sources=sources, libraries=libraries, include_dirs=['.'], - language='c++', + language=language, + define_macros=macros, ) -del sources, libraries +del sources, libraries, language, macros desc = 'MessagePack (de)serializer.' @@ -79,7 +87,6 @@ setup(name='msgpack-python', classifiers=[ 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', - 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', ] From 670bb3ca154d5f86fe72fe34a73d0ca8b7bd315a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 19 Aug 2012 02:53:16 +0900 Subject: [PATCH 0543/1172] Use C++ compiler on win32. --- setup.py | 69 +++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/setup.py b/setup.py index 7d8a5d4..074700b 100644 --- a/setup.py +++ b/setup.py @@ -4,23 +4,62 @@ version = (0, 2, 1, 'dev1') import os import sys +import shutil from glob import glob from distutils.command.sdist import sdist from setuptools import setup, Extension +from distutils.command.build_ext import build_ext + try: - from Cython.Distutils import build_ext import Cython.Compiler.Main as cython_compiler have_cython = True except ImportError: - from distutils.command.build_ext import build_ext have_cython = False + +def cythonize(src): + sys.stderr.write("cythonize: %r\n" % (src,)) + cython_compiler.compile([src]) + +def ensure_source(src): + pyx = os.path.splitext(src)[0] + '.pyx' + + if not os.path.exists(src): + if not have_cython: + raise Exception("""\ +Cython is required for building extension from checkout. +Install Cython >= 0.16 or install msgpack from PyPI. +""") + cythonize(src) + elif (os.path.exists(pyx) and + os.stat(src).st_mtime < os.stat(pyx).st_mtime and + have_cython): + cythonize(src) + + # Use C++ compiler on win32. + # MSVC9 doesn't provide stdint.h when using C Compiler. + if sys.platform == 'win32': + cpp = src + 'pp' + shutil.copy(src, cpp) + return cpp + else: + return src + + +class BuildExt(build_ext): + def build_extension(self, ext): + ext.sources = map(ensure_source, ext.sources) + return build_ext.build_extension(self, ext) + + # make msgpack/__verison__.py f = open('msgpack/__version__.py', 'w') -f.write("version = %r\n" % (version,)) -f.close() -del f +try: + f.write("version = %r\n" % (version,)) +finally: + f.close() + del f version_str = '.'.join(str(x) for x in version[:3]) if len(version) > 3 and version[3] != 'final': @@ -28,29 +67,18 @@ if len(version) > 3 and version[3] != 'final': # take care of extension modules. if have_cython: - sources = ['msgpack/_msgpack.pyx'] - class Sdist(sdist): def __init__(self, *args, **kwargs): - cy_opt = cython_compiler.default_options.copy() - #cy_opt['cplus'] = True for src in glob('msgpack/*.pyx'): - cython_compiler.compile(glob('msgpack/*.pyx'), cy_opt) + cythonize(src) sdist.__init__(self, *args, **kwargs) else: - sources = ['msgpack/_msgpack.c'] - - for f in sources: - if not os.path.exists(f): - raise ImportError("Building msgpack from VCS needs Cython. Install Cython or use sdist package.") - Sdist = sdist +sources = ['msgpack/_msgpack.c'] libraries = [] -language = 'c' if sys.platform == 'win32': libraries.append('ws2_32') - language = 'c++' if sys.byteorder == 'big': macros = [('__BIG_ENDIAN__', '1')] @@ -61,10 +89,9 @@ msgpack_mod = Extension('msgpack._msgpack', sources=sources, libraries=libraries, include_dirs=['.'], - language=language, define_macros=macros, ) -del sources, libraries, language, macros +del sources, libraries, macros desc = 'MessagePack (de)serializer.' @@ -77,7 +104,7 @@ setup(name='msgpack-python', author='INADA Naoki', author_email='songofacandy@gmail.com', version=version_str, - cmdclass={'build_ext': build_ext, 'sdist': Sdist}, + cmdclass={'build_ext': BuildExt, 'sdist': Sdist}, ext_modules=[msgpack_mod], packages=['msgpack'], description=desc, From 29b4b785d0152c964819bc613a0321214df78c47 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 19 Aug 2012 03:04:19 +0900 Subject: [PATCH 0544/1172] Fix build_ext doesn't work on Python 3. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 074700b..b03647a 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ Install Cython >= 0.16 or install msgpack from PyPI. class BuildExt(build_ext): def build_extension(self, ext): - ext.sources = map(ensure_source, ext.sources) + ext.sources = list(map(ensure_source, ext.sources)) return build_ext.build_extension(self, ext) From 814c42c2917ca62637ee37344c042174dfb10b4b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 19 Aug 2012 04:17:56 +0900 Subject: [PATCH 0545/1172] Change the way to manage version number. --- msgpack/__init__.py | 2 +- msgpack/_version.py | 1 + setup.py | 8 +------- 3 files changed, 3 insertions(+), 8 deletions(-) create mode 100644 msgpack/_version.py diff --git a/msgpack/__init__.py b/msgpack/__init__.py index cdf045f..98b1ab7 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,5 +1,5 @@ # coding: utf-8 -from msgpack.__version__ import * +from msgpack._version import version from msgpack._msgpack import * # alias for compatibility to simplejson/marshal/pickle. diff --git a/msgpack/_version.py b/msgpack/_version.py new file mode 100644 index 0000000..5355679 --- /dev/null +++ b/msgpack/_version.py @@ -0,0 +1 @@ +version = (0, 2, 1, 'dev1') diff --git a/setup.py b/setup.py index b03647a..6148dbd 100644 --- a/setup.py +++ b/setup.py @@ -53,13 +53,7 @@ class BuildExt(build_ext): return build_ext.build_extension(self, ext) -# make msgpack/__verison__.py -f = open('msgpack/__version__.py', 'w') -try: - f.write("version = %r\n" % (version,)) -finally: - f.close() - del f +exec(open('msgpack/_version.py').read()) version_str = '.'.join(str(x) for x in version[:3]) if len(version) > 3 and version[3] != 'final': From f74ce3caaa5d1aa3432dca1558a3d13a15d3fae3 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 20 Aug 2012 00:11:38 +0900 Subject: [PATCH 0546/1172] 0.2.1 --- ChangeLog.rst | 5 ++++- msgpack/_version.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index e86e3bd..c328f35 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,6 @@ 0.2.1 ======= -:release date: NOT RELEASED YET +:release date: 2012-08-20 Changes ------- @@ -12,6 +12,9 @@ Changes Bugs fixed ---------- +* Fix msgpack didn't work on SPARC Solaris. It was because choosing wrong byteorder + on compilation time. Use ``sys.byteorder`` to get correct byte order. + Very thanks to Chris Casey for giving test environment to me. 0.2.0 diff --git a/msgpack/_version.py b/msgpack/_version.py index 5355679..68ae707 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 2, 1, 'dev1') +version = (0, 2, 1) From e63a9437538919c669317b695d842807803301fe Mon Sep 17 00:00:00 2001 From: TobiasSimon Date: Mon, 20 Aug 2012 21:56:55 +0200 Subject: [PATCH 0547/1172] added float serialization support --- msgpack/_msgpack.pyx | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index ebf1592..62af8d3 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -30,6 +30,7 @@ cdef extern from "pack.h": int msgpack_pack_long(msgpack_packer* pk, long d) int msgpack_pack_long_long(msgpack_packer* pk, long long d) int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) + int msgpack_pack_float(msgpack_packer* pk, float d) int msgpack_pack_double(msgpack_packer* pk, double d) int msgpack_pack_array(msgpack_packer* pk, size_t l) int msgpack_pack_map(msgpack_packer* pk, size_t l) @@ -58,6 +59,7 @@ cdef class Packer(object): cdef object _berrors cdef char *encoding cdef char *unicode_errors + cdef bool use_float def __cinit__(self): cdef int buf_size = 1024*1024 @@ -67,7 +69,8 @@ cdef class Packer(object): self.pk.buf_size = buf_size self.pk.length = 0 - def __init__(self, default=None, encoding='utf-8', unicode_errors='strict'): + def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_float=False): + self.use_float = use_float if default is not None: if not PyCallable_Check(default): raise TypeError("default must be a callable.") @@ -94,7 +97,8 @@ cdef class Packer(object): cdef long long llval cdef unsigned long long ullval cdef long longval - cdef double fval + cdef float fval + cdef double dval cdef char* rawval cdef int ret cdef dict d @@ -120,8 +124,12 @@ cdef class Packer(object): longval = o ret = msgpack_pack_long(&self.pk, longval) elif PyFloat_Check(o): - fval = o - ret = msgpack_pack_double(&self.pk, fval) + if self.use_float: + fval = o + ret = msgpack_pack_float(&self.pk, fval) + else: + dval = o + ret = msgpack_pack_double(&self.pk, dval) elif PyBytes_Check(o): rawval = o ret = msgpack_pack_raw(&self.pk, len(o)) From 6aa4aead31e01e22f5297fadc7c3e4b0e90ba8ae Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 21 Aug 2012 14:56:32 +0900 Subject: [PATCH 0548/1172] Fix build from pyx doesn't work. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 6148dbd..2a20e16 100644 --- a/setup.py +++ b/setup.py @@ -31,11 +31,11 @@ def ensure_source(src): Cython is required for building extension from checkout. Install Cython >= 0.16 or install msgpack from PyPI. """) - cythonize(src) + cythonize(pyx) elif (os.path.exists(pyx) and os.stat(src).st_mtime < os.stat(pyx).st_mtime and have_cython): - cythonize(src) + cythonize(pyx) # Use C++ compiler on win32. # MSVC9 doesn't provide stdint.h when using C Compiler. From 235b928be7b43f1a5f1753ff08e685cbcf55bba4 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 24 Aug 2012 09:53:18 +0900 Subject: [PATCH 0549/1172] Stop disable/enable gc. json and pickle modules don't stop gc. It's a very dirty hack. --- msgpack/_msgpack.pyx | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index ebf1592..7ff0cff 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -13,10 +13,6 @@ from libc.string cimport * from libc.limits cimport * -import gc -_gc_disable = gc.disable -_gc_enable = gc.enable - cdef extern from "pack.h": struct msgpack_packer: char* buf @@ -242,11 +238,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, if not PyCallable_Check(list_hook): raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook - _gc_disable() - try: - ret = template_execute(&ctx, buf, buf_len, &off) - finally: - _gc_enable() + ret = template_execute(&ctx, buf, buf_len, &off) if ret == 1: return template_data(&ctx) else: @@ -444,9 +436,7 @@ cdef class Unpacker(object): """unpack one object""" cdef int ret while 1: - _gc_disable() ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) - _gc_enable() if ret == 1: o = template_data(&self.ctx) template_init(&self.ctx) From 56ec7ee1b1bdca7dd528cb6337f6b391581aefd0 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 24 Aug 2012 10:05:38 +0900 Subject: [PATCH 0550/1172] Update changelog --- ChangeLog.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index c328f35..67eac68 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,13 @@ +0.2.2 +======= +:release date: NOT RELEASED YET + +Bugs fixed +----------- +* ``unpack()`` didn't restores gc state when it called with gc disabled. + ``unpack()`` doesn't control gc now instead of restoring gc state collectly. + User can control gc state when gc cause performance issue. + 0.2.1 ======= :release date: 2012-08-20 From 0297b36bda332fe21ab7d4c4c549cc68ccd344bc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 21 Sep 2012 13:58:56 +0900 Subject: [PATCH 0551/1172] Fix reading more than read_size. --- msgpack/_msgpack.pyx | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 7ff0cff..15bf5a7 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -278,8 +278,8 @@ cdef class Unpacker(object): `unicode_errors` is used for decoding bytes. - `max_buffer_size` limits size of data waiting unpacked. 0 means unlimited - (default). + `max_buffer_size` limits size of data waiting unpacked. + 0 means system's INT_MAX (default). Raises `BufferFull` exception when it is insufficient. You shoud set this parameter when unpacking data from untrasted source. @@ -332,11 +332,11 @@ cdef class Unpacker(object): raise ValueError("`file_like.read` must be a callable.") if not max_buffer_size: max_buffer_size = INT_MAX + if read_size > max_buffer_size: + raise ValueError("read_size should be less or equal to max_buffer_size") if not read_size: read_size = min(max_buffer_size, 1024**2) self.max_buffer_size = max_buffer_size - if read_size > max_buffer_size: - raise ValueError("read_size should be less or equal to max_buffer_size") self.read_size = read_size self.buf = malloc(read_size) if self.buf == NULL: @@ -419,18 +419,15 @@ cdef class Unpacker(object): self.buf_size = buf_size self.buf_tail = tail + _buf_len - # prepare self.buf from file_like - cdef fill_buffer(self): - if self.file_like is not None: - next_bytes = self.file_like_read( - max(self.read_size, - self.max_buffer_size - (self.buf_tail - self.buf_head) - )) - if next_bytes: - self.append_buffer(PyBytes_AsString(next_bytes), - PyBytes_Size(next_bytes)) - else: - self.file_like = None + cdef read_from_file(self): + next_bytes = self.file_like_read( + min(self.read_size, + self.max_buffer_size - (self.buf_tail - self.buf_head) + )) + if next_bytes: + self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes)) + else: + self.file_like = None cpdef unpack(self): """unpack one object""" @@ -443,7 +440,7 @@ cdef class Unpacker(object): return o elif ret == 0: if self.file_like is not None: - self.fill_buffer() + self.read_from_file() continue raise StopIteration("No more unpack data.") else: From 397d772e110ad7000e0952d41c74d8efd322f01f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 21 Sep 2012 14:08:34 +0900 Subject: [PATCH 0552/1172] Rename use_float to use_single_float. --- msgpack/_msgpack.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index aef1228..af005cb 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -65,8 +65,8 @@ cdef class Packer(object): self.pk.buf_size = buf_size self.pk.length = 0 - def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_float=False): - self.use_float = use_float + def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False): + self.use_float = use_single_float if default is not None: if not PyCallable_Check(default): raise TypeError("default must be a callable.") From 51335bbee4502ac3af81363a10ef6718439377d1 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 21 Sep 2012 14:15:30 +0900 Subject: [PATCH 0553/1172] packb supports use_single_float option. --- ChangeLog.rst | 9 ++++++++- msgpack/_msgpack.pyx | 5 +++-- test/test_pack.py | 5 +++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 67eac68..4fd5cbc 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,11 @@ 0.2.2 ======= -:release date: NOT RELEASED YET +:release date: 2012-09-21 + +Changes +------- +* Add ``use_single_float`` option to ``Packer``. When it is true, packs float + object in single precision format. Bugs fixed ----------- @@ -8,6 +13,8 @@ Bugs fixed ``unpack()`` doesn't control gc now instead of restoring gc state collectly. User can control gc state when gc cause performance issue. +* ``Unpacker``'s ``read_size`` option didn't used. + 0.2.1 ======= :release date: 2012-08-20 diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index af005cb..c9f5e31 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -177,10 +177,11 @@ def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors) stream.write(packer.pack(o)) -def packb(object o, default=None, encoding='utf-8', unicode_errors='strict'): +def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False): """ pack o and return packed bytes.""" - packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors) + packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors, + use_single_float=use_single_float) return packer.pack(o) diff --git a/test/test_pack.py b/test/test_pack.py index 898cdb9..85d11a0 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -2,6 +2,7 @@ # coding: utf-8 import six +import struct from nose import main from nose.tools import * from nose.plugins.skip import SkipTest @@ -86,5 +87,9 @@ def testDecodeBinary(): re = unpackb(packb("abc"), encoding=None) assert_equal(re, b"abc") +def testPackFloat(): + assert_equal(packb(1.0, use_single_float=True), b'\xca' + struct.pack('>f', 1.0)) + assert_equal(packb(1.0, use_single_float=False), b'\xcb' + struct.pack('>d', 1.0)) + if __name__ == '__main__': main() From be405ec5cfe98a42283055baa901ec31de2632ba Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 21 Sep 2012 14:16:40 +0900 Subject: [PATCH 0554/1172] 0.2.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2a20e16..ae72221 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 2, 1, 'dev1') +version = (0, 2, 2) import os import sys From 5b66edaa156c43793b6f68013a738f545885b8d6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 21 Sep 2012 14:17:34 +0900 Subject: [PATCH 0555/1172] 0.2.2 (again) --- msgpack/_version.py | 2 +- setup.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index 68ae707..f343b7a 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 2, 1) +version = (0, 2, 2) diff --git a/setup.py b/setup.py index ae72221..86b0b34 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,5 @@ #!/usr/bin/env python # coding: utf-8 -version = (0, 2, 2) - import os import sys import shutil From ffec10dff3839ae182cff3d9fff67ab3fe6165be Mon Sep 17 00:00:00 2001 From: jnothman Date: Fri, 21 Sep 2012 16:03:41 +1000 Subject: [PATCH 0556/1172] Expose packed stream with Unpacker.read_bytes() At present, Unpacker buffers reading from the stream, meaning the stream can no longer be read directly. Unpacker.read_bytes(n) provides access to the underlying data, allowing content of known size to be read without unpacking. --- msgpack/_msgpack.pyx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index c9f5e31..d7ea4b4 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -455,6 +455,18 @@ cdef class Unpacker(object): else: raise ValueError("Unpack failed: error = %d" % (ret,)) + def read_bytes(self, Py_ssize_t nbytes): + """read a specified number of raw bytes from the stream""" + cdef size_t nread + ret = '' + while len(ret) < nbytes and self.file_like is not None: + if self.buf_head == self.buf_tail: + self.fill_buffer() + nread = min(self.buf_tail - self.buf_head, nbytes - len(ret)) + ret += PyBytes_FromStringAndSize(self.buf + self.buf_head, nread) + self.buf_head += nread + return ret + def __iter__(self): return self From 28058fb53d21097947d190bcc47e3609a6794e7a Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Sat, 22 Sep 2012 22:57:00 +1000 Subject: [PATCH 0557/1172] A first implementation of Unpacker.skip() --- msgpack/_msgpack.pyx | 25 +++++++++++++++++-------- msgpack/unpack_template.h | 24 ++++++++++++++---------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 80d34ab..c1e3e75 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -200,7 +200,7 @@ cdef extern from "unpack.h": PyObject* key int template_execute(template_context* ctx, const_char_ptr data, - size_t len, size_t* off) except -1 + size_t len, size_t* off, bool construct) except -1 void template_init(template_context* ctx) object template_data(template_context* ctx) @@ -246,7 +246,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, if not PyCallable_Check(list_hook): raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook - ret = template_execute(&ctx, buf, buf_len, &off) + ret = template_execute(&ctx, buf, buf_len, &off, True) if ret == 1: return template_data(&ctx) else: @@ -440,15 +440,12 @@ cdef class Unpacker(object): else: self.file_like = None - cpdef unpack(self): - """unpack one object""" + cpdef _unpack(self, bool construct): cdef int ret while 1: - ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head, construct) if ret == 1: - o = template_data(&self.ctx) - template_init(&self.ctx) - return o + return elif ret == 0: if self.file_like is not None: self.fill_buffer() @@ -457,6 +454,18 @@ cdef class Unpacker(object): else: raise ValueError("Unpack failed: error = %d" % (ret,)) + cpdef unpack(self): + """unpack one object""" + self._unpack(True) + o = template_data(&self.ctx) + template_init(&self.ctx) + + + cpdef skip(self): + """read and ignore one object, returning None""" + self._unpack(False) + template_init(&self.ctx) + def __iter__(self): return self diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index b844a24..10e41e1 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -95,7 +95,7 @@ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context } -msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) +msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off, bool construct) { assert(len >= *off); @@ -117,14 +117,17 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c int ret; +#define construct_cb(name) \ + construct && msgpack_unpack_callback(name) + #define push_simple_value(func) \ - if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \ + if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \ goto _push #define push_fixed_value(func, arg) \ - if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \ + if(construct_cb(func)(user, arg, &obj) < 0) { goto _failed; } \ goto _push #define push_variable_value(func, base, pos, len) \ - if(msgpack_unpack_callback(func)(user, \ + if(construct_cb(func)(user, \ (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ goto _push @@ -140,9 +143,9 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c #define start_container(func, count_, ct_) \ if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \ - if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ + if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ if((count_) == 0) { obj = stack[top].obj; \ - msgpack_unpack_callback(func##_end)(user, &obj); \ + construct_cb(func##_end)(user, &obj); \ goto _push; } \ stack[top].ct = ct_; \ stack[top].size = count_; \ @@ -340,10 +343,10 @@ _push: c = &stack[top-1]; switch(c->ct) { case CT_ARRAY_ITEM: - if(msgpack_unpack_callback(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } + if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } if(++c->count == c->size) { obj = c->obj; - msgpack_unpack_callback(_array_end)(user, &obj); + construct_cb(_array_end)(user, &obj); --top; /*printf("stack pop %d\n", top);*/ goto _push; @@ -354,10 +357,10 @@ _push: c->ct = CT_MAP_VALUE; goto _header_again; case CT_MAP_VALUE: - if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } + if(construct_cb(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } if(++c->count == c->size) { obj = c->obj; - msgpack_unpack_callback(_map_end)(user, &obj); + construct_cb(_map_end)(user, &obj); --top; /*printf("stack pop %d\n", top);*/ goto _push; @@ -399,6 +402,7 @@ _end: *off = p - (const unsigned char*)data; return ret; +#undef construct_cb } From 4d643894a1ab02b0836245b8a456200cac5ae314 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 02:13:32 +0900 Subject: [PATCH 0558/1172] Support packing subclass of dict. --- ChangeLog.rst | 11 +++++++++++ msgpack/_msgpack.pyx | 16 +++++++++++++--- test/test_pack.py | 26 ++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 4fd5cbc..46b83ee 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,14 @@ +0.2.3 +======= +:release date: in development + +Changes +------- + +Bugs fixed +----------- +* Can't pack subclass of dict. + 0.2.2 ======= :release date: 2012-09-21 diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index c9f5e31..976871e 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -139,11 +139,19 @@ cdef class Packer(object): ret = msgpack_pack_raw(&self.pk, len(o)) if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) - elif PyDict_Check(o): + elif PyDict_CheckExact(o): d = o ret = msgpack_pack_map(&self.pk, len(d)) if ret == 0: - for k,v in d.iteritems(): + for k, v in d.iteritems(): + ret = self._pack(k, nest_limit-1) + if ret != 0: break + ret = self._pack(v, nest_limit-1) + if ret != 0: break + elif PyDict_Check(o): + ret = msgpack_pack_map(&self.pk, len(o)) + if ret == 0: + for k, v in o.items(): ret = self._pack(k, nest_limit-1) if ret != 0: break ret = self._pack(v, nest_limit-1) @@ -332,7 +340,9 @@ cdef class Unpacker(object): def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, object object_hook=None, object list_hook=None, - encoding=None, unicode_errors='strict', int max_buffer_size=0): + encoding=None, unicode_errors='strict', int max_buffer_size=0, + object object_pairs_hook=None, + ): self.use_list = use_list self.file_like = file_like if file_like: diff --git a/test/test_pack.py b/test/test_pack.py index 85d11a0..b216c46 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -91,5 +91,31 @@ def testPackFloat(): assert_equal(packb(1.0, use_single_float=True), b'\xca' + struct.pack('>f', 1.0)) assert_equal(packb(1.0, use_single_float=False), b'\xcb' + struct.pack('>d', 1.0)) + +class odict(dict): + '''Reimplement OrderedDict to run test on Python 2.6''' + def __init__(self, seq): + self._seq = seq + dict.__init__(self, seq) + + def items(self): + return self._seq[:] + + def iteritems(self): + return iter(self._seq) + + def keys(self): + return [x[0] for x in self._seq] + +def test_odict(): + seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)] + od = odict(seq) + assert_equal(unpackb(packb(od)), dict(seq)) + # After object_pairs_hook is implemented. + #def pair_hook(seq): + # return seq + #assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook), seq) + + if __name__ == '__main__': main() From 8b2959bc0ab086a3dbe47176b3c241dd1a1ecf6c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 03:39:14 +0900 Subject: [PATCH 0559/1172] pack and packb raises ValueError when extra data passed. --- msgpack/_msgpack.pyx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 976871e..12ee2ea 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -257,7 +257,10 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, ctx.user.list_hook = list_hook ret = template_execute(&ctx, buf, buf_len, &off) if ret == 1: - return template_data(&ctx) + obj = template_data(&ctx) + if off < buf_len: + raise ValueError("Extra data.") + return obj else: return None @@ -461,7 +464,7 @@ cdef class Unpacker(object): if self.file_like is not None: self.read_from_file() continue - raise StopIteration("No more unpack data.") + raise StopIteration("No more data to unpack.") else: raise ValueError("Unpack failed: error = %d" % (ret,)) From 36b88b407718cb7b96f7ae12cffbb0c8a27e493b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 03:44:41 +0900 Subject: [PATCH 0560/1172] Add Roadmap. --- ROADMAP.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 ROADMAP.md diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..5245cc0 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,7 @@ +0.2 series +========== +Improve compatibility to simplejson. + +0.3 series +========== +Add features msgpack-ruby has. From e8842efdedb1917a28147aa8ad1bf6f7b729a751 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 08:57:32 +0900 Subject: [PATCH 0561/1172] Add py33 to tox. --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 5e80dd0..214c4c4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,6 @@ [tox] -envlist = py26,py27,py32 +envlist = py26,py27,py32,py33 + [testenv] deps= nose From eaf9891b4255f3b1ca5cf2ea5b631091523b913d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 10:00:18 +0900 Subject: [PATCH 0562/1172] clean some cython code. --- msgpack/_msgpack.pyx | 28 +++++++++++++++------------- msgpack/unpack_template.h | 2 +- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 9061d42..c8ee7bb 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -209,7 +209,7 @@ cdef extern from "unpack.h": PyObject* key int template_execute(template_context* ctx, const_char_ptr data, - size_t len, size_t* off, bool construct) except -1 + size_t len, size_t* off, bint construct) except -1 void template_init(template_context* ctx) object template_data(template_context* ctx) @@ -255,7 +255,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, if not PyCallable_Check(list_hook): raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook - ret = template_execute(&ctx, buf, buf_len, &off, True) + ret = template_execute(&ctx, buf, buf_len, &off, 1) if ret == 1: obj = template_data(&ctx) if off < buf_len: @@ -451,12 +451,18 @@ cdef class Unpacker(object): else: self.file_like = None - cpdef _unpack(self, bool construct): + cdef _unpack(self, bint construct): cdef int ret + cdef object obj while 1: ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head, construct) if ret == 1: - return + if construct: + obj = template_data(&self.ctx) + else: + obj = None + template_init(&self.ctx) + return obj elif ret == 0: if self.file_like is not None: self.read_from_file() @@ -465,23 +471,19 @@ cdef class Unpacker(object): else: raise ValueError("Unpack failed: error = %d" % (ret,)) - cpdef unpack(self): + def unpack(self): """unpack one object""" - self._unpack(True) - o = template_data(&self.ctx) - template_init(&self.ctx) + return self._unpack(1) - - cpdef skip(self): + def skip(self): """read and ignore one object, returning None""" - self._unpack(False) - template_init(&self.ctx) + return self._unpack(0) def __iter__(self): return self def __next__(self): - return self.unpack() + return self._unpack(1) # for debug. #def _buf(self): diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 10e41e1..5495a51 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -95,7 +95,7 @@ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context } -msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off, bool construct) +msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off, int construct) { assert(len >= *off); From 7d142d2bef0805a528f1cd84e173579209298380 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 10:02:11 +0900 Subject: [PATCH 0563/1172] Add changelog --- ChangeLog.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 46b83ee..fe5b820 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,12 @@ +0.3.0 +===== +:release date: in development + +Changes +------- +* Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman) + + 0.2.3 ======= :release date: in development From 48d693c1b9613fd976a3bf668f692ec22ad4a520 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 10:09:51 +0900 Subject: [PATCH 0564/1172] Add test for `.skip()` --- msgpack/_msgpack.pyx | 2 +- test/test_sequnpack.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index c8ee7bb..8d37aaa 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -451,7 +451,7 @@ cdef class Unpacker(object): else: self.file_like = None - cdef _unpack(self, bint construct): + cdef object _unpack(self, bint construct): cdef int ret cdef object obj while 1: diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index b1b80b2..aa47d3c 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -28,6 +28,20 @@ def test_foobar(): k += 1 assert k == len(b'foobar') +def test_foobar_skip(): + unpacker = Unpacker(read_size=3) + unpacker.feed(b'foobar') + assert unpacker.unpack() == ord(b'f') + unpacker.skip() + assert unpacker.unpack() == ord(b'o') + unpacker.skip() + assert unpacker.unpack() == ord(b'a') + unpacker.skip() + try: + o = unpacker.unpack() + assert 0, "should raise exception" + except StopIteration: + assert 1, "ok" def test_maxbuffersize(): nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3) From c3da8458681fc479233910d4c92dc84374e5efed Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 23 Sep 2012 11:16:59 +0900 Subject: [PATCH 0565/1172] Add docstring about raising ValueError when there are extra bytes. --- msgpack/_msgpack.pyx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 12ee2ea..0886580 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -217,8 +217,10 @@ cdef extern from "unpack.h": def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict", ): + """Unpack packed_bytes to object. Returns an unpacked object. + + Raises `ValueError` when `packed` contains extra bytes. """ - Unpack packed_bytes to object. Returns an unpacked object.""" cdef template_context ctx cdef size_t off = 0 cdef int ret @@ -268,14 +270,16 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict", ): - """ - unpack an object from stream. + """Unpack an object from `stream`. + + Raises `ValueError` when `stream` has extra bytes. """ return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors, ) + cdef class Unpacker(object): """ Streaming unpacker. From b06ed8eb75563111ef88a119f9f7a45e67f61736 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Sun, 23 Sep 2012 18:11:49 +1000 Subject: [PATCH 0566/1172] Factor context initialisation from unpackb and Unpacker --- msgpack/_msgpack.pyx | 82 +++++++++++++++----------------------------- 1 file changed, 28 insertions(+), 54 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index e0a1043..823ed62 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -213,6 +213,32 @@ cdef extern from "unpack.h": void template_init(template_context* ctx) object template_data(template_context* ctx) +cdef inline init_ctx(template_context *ctx, object object_hook, object list_hook, bint use_list, encoding, unicode_errors): + template_init(ctx) + ctx.user.use_list = use_list + ctx.user.object_hook = ctx.user.list_hook = NULL + if object_hook is not None: + if not PyCallable_Check(object_hook): + raise TypeError("object_hook must be a callable.") + ctx.user.object_hook = object_hook + if list_hook is not None: + if not PyCallable_Check(list_hook): + raise TypeError("list_hook must be a callable.") + ctx.user.list_hook = list_hook + if encoding is None: + ctx.user.encoding = NULL + ctx.user.unicode_errors = NULL + else: + if isinstance(encoding, unicode): + _bencoding = encoding.encode('ascii') + else: + _bencoding = encoding + ctx.user.encoding = PyBytes_AsString(_bencoding) + if isinstance(unicode_errors, unicode): + _berrors = unicode_errors.encode('ascii') + else: + _berrors = unicode_errors + ctx.user.unicode_errors = PyBytes_AsString(_berrors) def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict", @@ -229,34 +255,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef Py_ssize_t buf_len PyObject_AsReadBuffer(packed, &buf, &buf_len) - if encoding is None: - enc = NULL - err = NULL - else: - if isinstance(encoding, unicode): - bencoding = encoding.encode('ascii') - else: - bencoding = encoding - if isinstance(unicode_errors, unicode): - berrors = unicode_errors.encode('ascii') - else: - berrors = unicode_errors - enc = PyBytes_AsString(bencoding) - err = PyBytes_AsString(berrors) - - template_init(&ctx) - ctx.user.use_list = use_list - ctx.user.object_hook = ctx.user.list_hook = NULL - ctx.user.encoding = enc - ctx.user.unicode_errors = err - if object_hook is not None: - if not PyCallable_Check(object_hook): - raise TypeError("object_hook must be a callable.") - ctx.user.object_hook = object_hook - if list_hook is not None: - if not PyCallable_Check(list_hook): - raise TypeError("list_hook must be a callable.") - ctx.user.list_hook = list_hook + init_ctx(&ctx, object_hook, list_hook, use_list, encoding, unicode_errors) ret = template_execute(&ctx, buf, buf_len, &off, 1) if ret == 1: obj = template_data(&ctx) @@ -348,7 +347,6 @@ cdef class Unpacker(object): def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, object object_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, - object object_pairs_hook=None, ): self.use_list = use_list self.file_like = file_like @@ -370,31 +368,7 @@ cdef class Unpacker(object): self.buf_size = read_size self.buf_head = 0 self.buf_tail = 0 - template_init(&self.ctx) - self.ctx.user.use_list = use_list - self.ctx.user.object_hook = self.ctx.user.list_hook = NULL - if object_hook is not None: - if not PyCallable_Check(object_hook): - raise TypeError("object_hook must be a callable.") - self.ctx.user.object_hook = object_hook - if list_hook is not None: - if not PyCallable_Check(list_hook): - raise TypeError("list_hook must be a callable.") - self.ctx.user.list_hook = list_hook - if encoding is None: - self.ctx.user.encoding = NULL - self.ctx.user.unicode_errors = NULL - else: - if isinstance(encoding, unicode): - self._bencoding = encoding.encode('ascii') - else: - self._bencoding = encoding - self.ctx.user.encoding = PyBytes_AsString(self._bencoding) - if isinstance(unicode_errors, unicode): - self._berrors = unicode_errors.encode('ascii') - else: - self._berrors = unicode_errors - self.ctx.user.unicode_errors = PyBytes_AsString(self._berrors) + init_ctx(&self.ctx, object_hook, list_hook, use_list, encoding, unicode_errors) def feed(self, object next_bytes): cdef char* buf From 77942514db0c5a80e9f3f9bcb1e1939ecc8705e6 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Sun, 23 Sep 2012 19:37:28 +1000 Subject: [PATCH 0567/1172] Implement object_pairs_hook --- msgpack/_msgpack.pyx | 34 ++++++++++++++++++++++++++-------- msgpack/unpack.h | 30 +++++++++++++++++++++--------- msgpack/unpack_template.h | 2 +- test/test_obj.py | 10 ++++++++++ test/test_pack.py | 7 +++---- 5 files changed, 61 insertions(+), 22 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 823ed62..b6d8e8b 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -197,6 +197,7 @@ cdef extern from "unpack.h": ctypedef struct msgpack_user: int use_list PyObject* object_hook + bint has_pairs_hook # call object_hook with k-v pairs PyObject* list_hook char *encoding char *unicode_errors @@ -213,18 +214,32 @@ cdef extern from "unpack.h": void template_init(template_context* ctx) object template_data(template_context* ctx) -cdef inline init_ctx(template_context *ctx, object object_hook, object list_hook, bint use_list, encoding, unicode_errors): +cdef inline init_ctx(template_context *ctx, object object_hook, object object_pairs_hook, object list_hook, bint use_list, encoding, unicode_errors): template_init(ctx) ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL + + if object_hook is not None and object_pairs_hook is not None: + raise ValueError("object_pairs_hook and object_hook are mutually exclusive.") + if object_hook is not None: if not PyCallable_Check(object_hook): raise TypeError("object_hook must be a callable.") ctx.user.object_hook = object_hook + + if object_pairs_hook is None: + ctx.user.has_pairs_hook = False + else: + if not PyCallable_Check(object_pairs_hook): + raise TypeError("object_pairs_hook must be a callable.") + ctx.user.object_hook = object_pairs_hook + ctx.user.has_pairs_hook = True + if list_hook is not None: if not PyCallable_Check(list_hook): raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook + if encoding is None: ctx.user.encoding = NULL ctx.user.unicode_errors = NULL @@ -240,7 +255,7 @@ cdef inline init_ctx(template_context *ctx, object object_hook, object list_hook _berrors = unicode_errors ctx.user.unicode_errors = PyBytes_AsString(_berrors) -def unpackb(object packed, object object_hook=None, object list_hook=None, +def unpackb(object packed, object object_hook=None, object object_pairs_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict", ): """Unpack packed_bytes to object. Returns an unpacked object. @@ -255,7 +270,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef Py_ssize_t buf_len PyObject_AsReadBuffer(packed, &buf, &buf_len) - init_ctx(&ctx, object_hook, list_hook, use_list, encoding, unicode_errors) + init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors) ret = template_execute(&ctx, buf, buf_len, &off, 1) if ret == 1: obj = template_data(&ctx) @@ -266,7 +281,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, return None -def unpack(object stream, object object_hook=None, object list_hook=None, +def unpack(object stream, object object_hook=None, object object_pairs_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict", ): """Unpack an object from `stream`. @@ -274,7 +289,7 @@ def unpack(object stream, object object_hook=None, object list_hook=None, Raises `ValueError` when `stream` has extra bytes. """ return unpackb(stream.read(), use_list=use_list, - object_hook=object_hook, list_hook=list_hook, + object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors, ) @@ -294,7 +309,10 @@ cdef class Unpacker(object): Otherwise, it is deserialized to Python tuple. (default: False) `object_hook` is same to simplejson. If it is not None, it should be callable - and Unpacker calls it when deserializing key-value. + and Unpacker calls it with a dict argument after deserializing a map. + + `object_pairs_hook` is same to simplejson. If it is not None, it should be callable + and Unpacker calls it with a list of key-value pairs after deserializing a map. `encoding` is encoding used for decoding msgpack bytes. If it is None (default), msgpack bytes is deserialized to Python bytes. @@ -345,7 +363,7 @@ cdef class Unpacker(object): self.buf = NULL def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, - object object_hook=None, object list_hook=None, + object object_hook=None, object object_pairs_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, ): self.use_list = use_list @@ -368,7 +386,7 @@ cdef class Unpacker(object): self.buf_size = read_size self.buf_head = 0 self.buf_tail = 0 - init_ctx(&self.ctx, object_hook, list_hook, use_list, encoding, unicode_errors) + init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors) def feed(self, object next_bytes): cdef char* buf diff --git a/msgpack/unpack.h b/msgpack/unpack.h index a106f9c..7064a1b 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -22,6 +22,7 @@ typedef struct unpack_user { int use_list; PyObject *object_hook; + bool has_pairs_hook; PyObject *list_hook; const char *encoding; const char *unicode_errors; @@ -160,9 +161,7 @@ static inline int template_callback_array_item(unpack_user* u, unsigned int curr static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_object* c) { if (u->list_hook) { - PyObject *arglist = Py_BuildValue("(O)", *c); - PyObject *new_c = PyEval_CallObject(u->list_hook, arglist); - Py_DECREF(arglist); + PyObject *new_c = PyEval_CallFunction(u->list_hook, "(O)", *c); Py_DECREF(*c); *c = new_c; } @@ -171,16 +170,31 @@ static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_obj static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - PyObject *p = PyDict_New(); + PyObject *p; + if (u->has_pairs_hook) { + p = PyList_New(n); // Or use tuple? + } + else { + p = PyDict_New(); + } if (!p) return -1; *o = p; return 0; } -static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) +static inline int template_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { - if (PyDict_SetItem(*c, k, v) == 0) { + if (u->has_pairs_hook) { + msgpack_unpack_object item = PyTuple_Pack(2, k, v); + if (!item) + return -1; + Py_DECREF(k); + Py_DECREF(v); + PyList_SET_ITEM(*c, current, item); + return 0; + } + else if (PyDict_SetItem(*c, k, v) == 0) { Py_DECREF(k); Py_DECREF(v); return 0; @@ -191,9 +205,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c) { if (u->object_hook) { - PyObject *arglist = Py_BuildValue("(O)", *c); - PyObject *new_c = PyEval_CallObject(u->object_hook, arglist); - Py_DECREF(arglist); + PyObject *new_c = PyEval_CallFunction(u->object_hook, "(O)", *c); Py_DECREF(*c); *c = new_c; } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 5495a51..6080a51 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -357,7 +357,7 @@ _push: c->ct = CT_MAP_VALUE; goto _header_again; case CT_MAP_VALUE: - if(construct_cb(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } + if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } if(++c->count == c->size) { obj = c->obj; construct_cb(_map_end)(user, &obj); diff --git a/test/test_obj.py b/test/test_obj.py index d155b73..e0d89fc 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -26,6 +26,16 @@ def test_decode_hook(): unpacked = unpackb(packed, object_hook=_decode_complex) eq_(unpacked[1], 1+2j) +def test_decode_pairs_hook(): + packed = packb([3, {1: 2, 3: 4}]) + prod_sum = 1 * 2 + 3 * 4 + unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l)) + eq_(unpacked[1], prod_sum) + +@raises(ValueError) +def test_only_one_obj_hook(): + unpackb('', object_hook=lambda x: x, object_pairs_hook=lambda x: x) + @raises(ValueError) def test_bad_hook(): packed = packb([3, 1+2j], default=lambda o: o) diff --git a/test/test_pack.py b/test/test_pack.py index b216c46..2c99873 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -111,10 +111,9 @@ def test_odict(): seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)] od = odict(seq) assert_equal(unpackb(packb(od)), dict(seq)) - # After object_pairs_hook is implemented. - #def pair_hook(seq): - # return seq - #assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook), seq) + def pair_hook(seq): + return seq + assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook), seq) if __name__ == '__main__': From e7c51d9089e9270ce197c00a6af1c60e45f36e97 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Sun, 23 Sep 2012 11:13:44 +1000 Subject: [PATCH 0568/1172] Cleaner read_bytes and a test case No longer reads via buffer for unbuffered bytes --- msgpack/_msgpack.pyx | 12 +++++------- test/test_sequnpack.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index d7ea4b4..30fb9fc 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -458,13 +458,11 @@ cdef class Unpacker(object): def read_bytes(self, Py_ssize_t nbytes): """read a specified number of raw bytes from the stream""" cdef size_t nread - ret = '' - while len(ret) < nbytes and self.file_like is not None: - if self.buf_head == self.buf_tail: - self.fill_buffer() - nread = min(self.buf_tail - self.buf_head, nbytes - len(ret)) - ret += PyBytes_FromStringAndSize(self.buf + self.buf_head, nread) - self.buf_head += nread + nread = min(self.buf_tail - self.buf_head, nbytes) + ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread) + self.buf_head += nread + if len(ret) < nbytes and self.file_like is not None: + ret += self.file_like.read(nbytes - len(ret)) return ret def __iter__(self): diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index b1b80b2..c763f40 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # coding: utf-8 +import six from msgpack import Unpacker, BufferFull import nose @@ -42,5 +43,20 @@ def test_maxbuffersize(): assert ord('b') == next(unpacker) +def test_readbytes(): + unpacker = Unpacker(read_size=3) + unpacker.feed(b'foobar') + assert unpacker.unpack() == ord(b'f') + assert unpacker.read_bytes(3) == b'oob' + assert unpacker.unpack() == ord(b'a') + assert unpacker.unpack() == ord(b'r') + + # Test buffer refill + unpacker = Unpacker(six.BytesIO(b'foobar'), read_size=3) + assert unpacker.unpack() == ord(b'f') + assert unpacker.read_bytes(3) == b'oob' + assert unpacker.unpack() == ord(b'a') + assert unpacker.unpack() == ord(b'r') + if __name__ == '__main__': nose.main() From 60df5eadaf507594b73e5e5a887da1fc52cb3f32 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 02:12:55 +0900 Subject: [PATCH 0569/1172] Warn when use_list is not specified. --- msgpack/_msgpack.pyx | 30 ++++++++++++++++++++++-------- setup.py | 2 +- test/test_buffer.py | 4 ++-- test/test_format.py | 4 ++-- test/test_pack.py | 12 ++++++------ test/test_seq.py | 2 +- test/test_sequnpack.py | 6 +++--- 7 files changed, 37 insertions(+), 23 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index e0a1043..d0c4541 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -1,12 +1,16 @@ # coding: utf-8 #cython: embedsignature=True +import warnings + from cpython cimport * cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef char* const_void_ptr "const void*" ctypedef struct PyObject cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1 + char* __FILE__ + int __LINE__ from libc.stdlib cimport * from libc.string cimport * @@ -195,7 +199,7 @@ def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use cdef extern from "unpack.h": ctypedef struct msgpack_user: - int use_list + bint use_list PyObject* object_hook PyObject* list_hook char *encoding @@ -215,7 +219,7 @@ cdef extern from "unpack.h": def unpackb(object packed, object object_hook=None, object list_hook=None, - bint use_list=0, encoding=None, unicode_errors="strict", + use_list=None, encoding=None, unicode_errors="strict", ): """Unpack packed_bytes to object. Returns an unpacked object. @@ -227,6 +231,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef char* buf cdef Py_ssize_t buf_len + PyObject_AsReadBuffer(packed, &buf, &buf_len) if encoding is None: @@ -245,7 +250,11 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, err = PyBytes_AsString(berrors) template_init(&ctx) - ctx.user.use_list = use_list + if use_list is None: + warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) + ctx.user.use_list = 0 + else: + ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL ctx.user.encoding = enc ctx.user.unicode_errors = err @@ -268,12 +277,15 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, def unpack(object stream, object object_hook=None, object list_hook=None, - bint use_list=0, encoding=None, unicode_errors="strict", + use_list=None, encoding=None, unicode_errors="strict", ): """Unpack an object from `stream`. Raises `ValueError` when `stream` has extra bytes. """ + if use_list is None: + warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) + use_list = 0 return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors, @@ -292,7 +304,7 @@ cdef class Unpacker(object): (default: min(1024**2, max_buffer_size)) If `use_list` is true, msgpack list is deserialized to Python list. - Otherwise, it is deserialized to Python tuple. (default: False) + Otherwise, it is deserialized to Python tuple. `object_hook` is same to simplejson. If it is not None, it should be callable and Unpacker calls it when deserializing key-value. @@ -330,7 +342,6 @@ cdef class Unpacker(object): cdef object file_like cdef object file_like_read cdef Py_ssize_t read_size - cdef bint use_list cdef object object_hook cdef object _bencoding cdef object _berrors @@ -345,12 +356,15 @@ cdef class Unpacker(object): free(self.buf) self.buf = NULL - def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, + def __init__(self, file_like=None, Py_ssize_t read_size=0, use_list=None, object object_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, object object_pairs_hook=None, ): - self.use_list = use_list + if use_list is None: + warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) + use_list = 0 + self.file_like = file_like if file_like: self.file_like_read = file_like.read diff --git a/setup.py b/setup.py index 86b0b34..9f0ce5d 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ except ImportError: def cythonize(src): sys.stderr.write("cythonize: %r\n" % (src,)) - cython_compiler.compile([src]) + cython_compiler.compile([src], emit_linenums=True) def ensure_source(src): pyx = os.path.splitext(src)[0] + '.pyx' diff --git a/test/test_buffer.py b/test/test_buffer.py index 01310a0..785fb60 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -9,8 +9,8 @@ def test_unpack_buffer(): from array import array buf = array('b') buf.fromstring(packb(('foo', 'bar'))) - obj = unpackb(buf) - assert_equal((b'foo', b'bar'), obj) + obj = unpackb(buf, use_list=1) + assert_equal([b'foo', b'bar'], obj) if __name__ == '__main__': main() diff --git a/test/test_format.py b/test/test_format.py index c03b3e2..ac08709 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -5,8 +5,8 @@ from nose import main from nose.tools import * from msgpack import unpackb -def check(src, should): - assert_equal(unpackb(src), should) +def check(src, should, use_list=0): + assert_equal(unpackb(src, use_list=use_list), should) def testSimpleValue(): check(b"\x93\xc0\xc2\xc3", diff --git a/test/test_pack.py b/test/test_pack.py index b216c46..dc77dfe 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -11,8 +11,8 @@ from msgpack import packb, unpackb, Unpacker, Packer from io import BytesIO -def check(data): - re = unpackb(packb(data)) +def check(data, use_list=False): + re = unpackb(packb(data), use_list=use_list) assert_equal(re, data) def testPack(): @@ -34,7 +34,7 @@ def testPackUnicode(): six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("РуÑÑкий текÑÑ‚"), ] for td in test_data: - re = unpackb(packb(td, encoding='utf-8'), encoding='utf-8') + re = unpackb(packb(td, encoding='utf-8'), use_list=0, encoding='utf-8') assert_equal(re, td) packer = Packer(encoding='utf-8') data = packer.pack(td) @@ -46,11 +46,11 @@ def testPackUTF32(): test_data = [ six.u(""), six.u("abcd"), - (six.u("defgh"),), + [six.u("defgh")], six.u("РуÑÑкий текÑÑ‚"), ] for td in test_data: - re = unpackb(packb(td, encoding='utf-32'), encoding='utf-32') + re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32') assert_equal(re, td) except LookupError: raise SkipTest @@ -110,7 +110,7 @@ class odict(dict): def test_odict(): seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)] od = odict(seq) - assert_equal(unpackb(packb(od)), dict(seq)) + assert_equal(unpackb(packb(od), use_list=1), dict(seq)) # After object_pairs_hook is implemented. #def pair_hook(seq): # return seq diff --git a/test/test_seq.py b/test/test_seq.py index d0f9ccc..72e935a 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -34,7 +34,7 @@ def test_exceeding_unpacker_read_size(): f = io.BytesIO(dumpf.getvalue()) dumpf.close() - unpacker = msgpack.Unpacker(f, read_size=read_size) + unpacker = msgpack.Unpacker(f, read_size=read_size, use_list=1) read_count = 0 for idx, o in enumerate(unpacker): diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index aa47d3c..dac36a8 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -5,7 +5,7 @@ from msgpack import Unpacker, BufferFull import nose def test_foobar(): - unpacker = Unpacker(read_size=3) + unpacker = Unpacker(read_size=3, use_list=1) unpacker.feed(b'foobar') assert unpacker.unpack() == ord(b'f') assert unpacker.unpack() == ord(b'o') @@ -29,7 +29,7 @@ def test_foobar(): assert k == len(b'foobar') def test_foobar_skip(): - unpacker = Unpacker(read_size=3) + unpacker = Unpacker(read_size=3, use_list=1) unpacker.feed(b'foobar') assert unpacker.unpack() == ord(b'f') unpacker.skip() @@ -45,7 +45,7 @@ def test_foobar_skip(): def test_maxbuffersize(): nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3) - unpacker = Unpacker(read_size=3, max_buffer_size=3) + unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1) unpacker.feed(b'fo') nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob') unpacker.feed(b'o') From c2a2d417f1e3a95992b41c3b24e6470077c99c6f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 02:20:53 +0900 Subject: [PATCH 0570/1172] Fix warnings in tests. --- test/test_case.py | 6 +++--- test/test_obj.py | 8 ++++---- test/test_pack.py | 15 +++++++-------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/test/test_case.py b/test/test_case.py index b88714d..9cbf9bd 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -9,7 +9,7 @@ from msgpack import packb, unpackb def check(length, obj): v = packb(obj) assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) - assert_equal(unpackb(v), obj) + assert_equal(unpackb(v, use_list=0), obj) def test_1(): for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1, @@ -71,7 +71,7 @@ def test_array32(): def match(obj, buf): assert_equal(packb(obj), buf) - assert_equal(unpackb(buf), obj) + assert_equal(unpackb(buf, use_list=0), obj) def test_match(): cases = [ @@ -99,7 +99,7 @@ def test_match(): match(v, p) def test_unicode(): - assert_equal(b'foobar', unpackb(packb('foobar'))) + assert_equal(b'foobar', unpackb(packb('foobar'), use_list=1)) if __name__ == '__main__': main() diff --git a/test/test_obj.py b/test/test_obj.py index d155b73..d809093 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -18,25 +18,25 @@ def _encode_complex(obj): def test_encode_hook(): packed = packb([3, 1+2j], default=_encode_complex) - unpacked = unpackb(packed) + unpacked = unpackb(packed, use_list=1) eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2}) def test_decode_hook(): packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}]) - unpacked = unpackb(packed, object_hook=_decode_complex) + unpacked = unpackb(packed, object_hook=_decode_complex, use_list=1) eq_(unpacked[1], 1+2j) @raises(ValueError) def test_bad_hook(): packed = packb([3, 1+2j], default=lambda o: o) - unpacked = unpackb(packed) + unpacked = unpackb(packed, use_list=1) def _arr_to_str(arr): return ''.join(str(c) for c in arr) def test_array_hook(): packed = packb([1,2,3]) - unpacked = unpackb(packed, list_hook=_arr_to_str) + unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1) eq_(unpacked, '123') if __name__ == '__main__': diff --git a/test/test_pack.py b/test/test_pack.py index dc77dfe..9bd2b32 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -31,14 +31,14 @@ def testPack(): def testPackUnicode(): test_data = [ - six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("РуÑÑкий текÑÑ‚"), + six.u(""), six.u("abcd"), [six.u("defgh")], six.u("РуÑÑкий текÑÑ‚"), ] for td in test_data: - re = unpackb(packb(td, encoding='utf-8'), use_list=0, encoding='utf-8') + re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8') assert_equal(re, td) packer = Packer(encoding='utf-8') data = packer.pack(td) - re = Unpacker(BytesIO(data), encoding='utf-8').unpack() + re = Unpacker(BytesIO(data), encoding='utf-8', use_list=1).unpack() assert_equal(re, td) def testPackUTF32(): @@ -63,20 +63,19 @@ def testPackBytes(): check(td) def testIgnoreUnicodeErrors(): - re = unpackb(packb(b'abc\xeddef'), - encoding='utf-8', unicode_errors='ignore') + re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1) assert_equal(re, "abcdef") @raises(UnicodeDecodeError) def testStrictUnicodeUnpack(): - unpackb(packb(b'abc\xeddef'), encoding='utf-8') + unpackb(packb(b'abc\xeddef'), encoding='utf-8', use_list=1) @raises(UnicodeEncodeError) def testStrictUnicodePack(): packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict') def testIgnoreErrorsPack(): - re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8') + re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1) assert_equal(re, six.u("abcdef")) @raises(TypeError) @@ -84,7 +83,7 @@ def testNoEncoding(): packb(six.u("abc"), encoding=None) def testDecodeBinary(): - re = unpackb(packb("abc"), encoding=None) + re = unpackb(packb("abc"), encoding=None, use_list=1) assert_equal(re, b"abc") def testPackFloat(): From d503788e9537498ff2ed0da1f836dc4de6074981 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 02:12:55 +0900 Subject: [PATCH 0571/1172] Warn when use_list is not specified. Conflicts: test/test_sequnpack.py --- msgpack/_msgpack.pyx | 30 ++++++++++++++++++++++-------- setup.py | 2 +- test/test_buffer.py | 4 ++-- test/test_format.py | 4 ++-- test/test_pack.py | 12 ++++++------ test/test_seq.py | 2 +- test/test_sequnpack.py | 5 ++--- 7 files changed, 36 insertions(+), 23 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 0886580..44a50ae 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -1,12 +1,16 @@ # coding: utf-8 #cython: embedsignature=True +import warnings + from cpython cimport * cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef char* const_void_ptr "const void*" ctypedef struct PyObject cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1 + char* __FILE__ + int __LINE__ from libc.stdlib cimport * from libc.string cimport * @@ -195,7 +199,7 @@ def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use cdef extern from "unpack.h": ctypedef struct msgpack_user: - int use_list + bint use_list PyObject* object_hook PyObject* list_hook char *encoding @@ -215,7 +219,7 @@ cdef extern from "unpack.h": def unpackb(object packed, object object_hook=None, object list_hook=None, - bint use_list=0, encoding=None, unicode_errors="strict", + use_list=None, encoding=None, unicode_errors="strict", ): """Unpack packed_bytes to object. Returns an unpacked object. @@ -227,6 +231,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef char* buf cdef Py_ssize_t buf_len + PyObject_AsReadBuffer(packed, &buf, &buf_len) if encoding is None: @@ -245,7 +250,11 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, err = PyBytes_AsString(berrors) template_init(&ctx) - ctx.user.use_list = use_list + if use_list is None: + warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) + ctx.user.use_list = 0 + else: + ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL ctx.user.encoding = enc ctx.user.unicode_errors = err @@ -268,12 +277,15 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, def unpack(object stream, object object_hook=None, object list_hook=None, - bint use_list=0, encoding=None, unicode_errors="strict", + use_list=None, encoding=None, unicode_errors="strict", ): """Unpack an object from `stream`. Raises `ValueError` when `stream` has extra bytes. """ + if use_list is None: + warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) + use_list = 0 return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors, @@ -292,7 +304,7 @@ cdef class Unpacker(object): (default: min(1024**2, max_buffer_size)) If `use_list` is true, msgpack list is deserialized to Python list. - Otherwise, it is deserialized to Python tuple. (default: False) + Otherwise, it is deserialized to Python tuple. `object_hook` is same to simplejson. If it is not None, it should be callable and Unpacker calls it when deserializing key-value. @@ -330,7 +342,6 @@ cdef class Unpacker(object): cdef object file_like cdef object file_like_read cdef Py_ssize_t read_size - cdef bint use_list cdef object object_hook cdef object _bencoding cdef object _berrors @@ -345,12 +356,15 @@ cdef class Unpacker(object): free(self.buf) self.buf = NULL - def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0, + def __init__(self, file_like=None, Py_ssize_t read_size=0, use_list=None, object object_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, object object_pairs_hook=None, ): - self.use_list = use_list + if use_list is None: + warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) + use_list = 0 + self.file_like = file_like if file_like: self.file_like_read = file_like.read diff --git a/setup.py b/setup.py index 86b0b34..9f0ce5d 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ except ImportError: def cythonize(src): sys.stderr.write("cythonize: %r\n" % (src,)) - cython_compiler.compile([src]) + cython_compiler.compile([src], emit_linenums=True) def ensure_source(src): pyx = os.path.splitext(src)[0] + '.pyx' diff --git a/test/test_buffer.py b/test/test_buffer.py index 01310a0..785fb60 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -9,8 +9,8 @@ def test_unpack_buffer(): from array import array buf = array('b') buf.fromstring(packb(('foo', 'bar'))) - obj = unpackb(buf) - assert_equal((b'foo', b'bar'), obj) + obj = unpackb(buf, use_list=1) + assert_equal([b'foo', b'bar'], obj) if __name__ == '__main__': main() diff --git a/test/test_format.py b/test/test_format.py index c03b3e2..ac08709 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -5,8 +5,8 @@ from nose import main from nose.tools import * from msgpack import unpackb -def check(src, should): - assert_equal(unpackb(src), should) +def check(src, should, use_list=0): + assert_equal(unpackb(src, use_list=use_list), should) def testSimpleValue(): check(b"\x93\xc0\xc2\xc3", diff --git a/test/test_pack.py b/test/test_pack.py index b216c46..dc77dfe 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -11,8 +11,8 @@ from msgpack import packb, unpackb, Unpacker, Packer from io import BytesIO -def check(data): - re = unpackb(packb(data)) +def check(data, use_list=False): + re = unpackb(packb(data), use_list=use_list) assert_equal(re, data) def testPack(): @@ -34,7 +34,7 @@ def testPackUnicode(): six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("РуÑÑкий текÑÑ‚"), ] for td in test_data: - re = unpackb(packb(td, encoding='utf-8'), encoding='utf-8') + re = unpackb(packb(td, encoding='utf-8'), use_list=0, encoding='utf-8') assert_equal(re, td) packer = Packer(encoding='utf-8') data = packer.pack(td) @@ -46,11 +46,11 @@ def testPackUTF32(): test_data = [ six.u(""), six.u("abcd"), - (six.u("defgh"),), + [six.u("defgh")], six.u("РуÑÑкий текÑÑ‚"), ] for td in test_data: - re = unpackb(packb(td, encoding='utf-32'), encoding='utf-32') + re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32') assert_equal(re, td) except LookupError: raise SkipTest @@ -110,7 +110,7 @@ class odict(dict): def test_odict(): seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)] od = odict(seq) - assert_equal(unpackb(packb(od)), dict(seq)) + assert_equal(unpackb(packb(od), use_list=1), dict(seq)) # After object_pairs_hook is implemented. #def pair_hook(seq): # return seq diff --git a/test/test_seq.py b/test/test_seq.py index d0f9ccc..72e935a 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -34,7 +34,7 @@ def test_exceeding_unpacker_read_size(): f = io.BytesIO(dumpf.getvalue()) dumpf.close() - unpacker = msgpack.Unpacker(f, read_size=read_size) + unpacker = msgpack.Unpacker(f, read_size=read_size, use_list=1) read_count = 0 for idx, o in enumerate(unpacker): diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index b1b80b2..21fc3be 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -5,7 +5,7 @@ from msgpack import Unpacker, BufferFull import nose def test_foobar(): - unpacker = Unpacker(read_size=3) + unpacker = Unpacker(read_size=3, use_list=1) unpacker.feed(b'foobar') assert unpacker.unpack() == ord(b'f') assert unpacker.unpack() == ord(b'o') @@ -28,10 +28,9 @@ def test_foobar(): k += 1 assert k == len(b'foobar') - def test_maxbuffersize(): nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3) - unpacker = Unpacker(read_size=3, max_buffer_size=3) + unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1) unpacker.feed(b'fo') nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob') unpacker.feed(b'o') From c280e589884f2b4afd064cffd08b0f353db93036 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 02:20:53 +0900 Subject: [PATCH 0572/1172] Fix warnings in tests. --- test/test_case.py | 6 +++--- test/test_obj.py | 8 ++++---- test/test_pack.py | 15 +++++++-------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/test/test_case.py b/test/test_case.py index b88714d..9cbf9bd 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -9,7 +9,7 @@ from msgpack import packb, unpackb def check(length, obj): v = packb(obj) assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) - assert_equal(unpackb(v), obj) + assert_equal(unpackb(v, use_list=0), obj) def test_1(): for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1, @@ -71,7 +71,7 @@ def test_array32(): def match(obj, buf): assert_equal(packb(obj), buf) - assert_equal(unpackb(buf), obj) + assert_equal(unpackb(buf, use_list=0), obj) def test_match(): cases = [ @@ -99,7 +99,7 @@ def test_match(): match(v, p) def test_unicode(): - assert_equal(b'foobar', unpackb(packb('foobar'))) + assert_equal(b'foobar', unpackb(packb('foobar'), use_list=1)) if __name__ == '__main__': main() diff --git a/test/test_obj.py b/test/test_obj.py index d155b73..d809093 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -18,25 +18,25 @@ def _encode_complex(obj): def test_encode_hook(): packed = packb([3, 1+2j], default=_encode_complex) - unpacked = unpackb(packed) + unpacked = unpackb(packed, use_list=1) eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2}) def test_decode_hook(): packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}]) - unpacked = unpackb(packed, object_hook=_decode_complex) + unpacked = unpackb(packed, object_hook=_decode_complex, use_list=1) eq_(unpacked[1], 1+2j) @raises(ValueError) def test_bad_hook(): packed = packb([3, 1+2j], default=lambda o: o) - unpacked = unpackb(packed) + unpacked = unpackb(packed, use_list=1) def _arr_to_str(arr): return ''.join(str(c) for c in arr) def test_array_hook(): packed = packb([1,2,3]) - unpacked = unpackb(packed, list_hook=_arr_to_str) + unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1) eq_(unpacked, '123') if __name__ == '__main__': diff --git a/test/test_pack.py b/test/test_pack.py index dc77dfe..9bd2b32 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -31,14 +31,14 @@ def testPack(): def testPackUnicode(): test_data = [ - six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("РуÑÑкий текÑÑ‚"), + six.u(""), six.u("abcd"), [six.u("defgh")], six.u("РуÑÑкий текÑÑ‚"), ] for td in test_data: - re = unpackb(packb(td, encoding='utf-8'), use_list=0, encoding='utf-8') + re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8') assert_equal(re, td) packer = Packer(encoding='utf-8') data = packer.pack(td) - re = Unpacker(BytesIO(data), encoding='utf-8').unpack() + re = Unpacker(BytesIO(data), encoding='utf-8', use_list=1).unpack() assert_equal(re, td) def testPackUTF32(): @@ -63,20 +63,19 @@ def testPackBytes(): check(td) def testIgnoreUnicodeErrors(): - re = unpackb(packb(b'abc\xeddef'), - encoding='utf-8', unicode_errors='ignore') + re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1) assert_equal(re, "abcdef") @raises(UnicodeDecodeError) def testStrictUnicodeUnpack(): - unpackb(packb(b'abc\xeddef'), encoding='utf-8') + unpackb(packb(b'abc\xeddef'), encoding='utf-8', use_list=1) @raises(UnicodeEncodeError) def testStrictUnicodePack(): packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict') def testIgnoreErrorsPack(): - re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8') + re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1) assert_equal(re, six.u("abcdef")) @raises(TypeError) @@ -84,7 +83,7 @@ def testNoEncoding(): packb(six.u("abc"), encoding=None) def testDecodeBinary(): - re = unpackb(packb("abc"), encoding=None) + re = unpackb(packb("abc"), encoding=None, use_list=1) assert_equal(re, b"abc") def testPackFloat(): From 15a46eb143d979c7b55f30d12d8bfe6a846f2a7e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 02:42:38 +0900 Subject: [PATCH 0573/1172] use_list=1 is default --- msgpack/_msgpack.pyx | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index d0c4541..e932ba9 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -219,7 +219,7 @@ cdef extern from "unpack.h": def unpackb(object packed, object object_hook=None, object list_hook=None, - use_list=None, encoding=None, unicode_errors="strict", + bint use_list=1, encoding=None, unicode_errors="strict", ): """Unpack packed_bytes to object. Returns an unpacked object. @@ -250,11 +250,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, err = PyBytes_AsString(berrors) template_init(&ctx) - if use_list is None: - warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) - ctx.user.use_list = 0 - else: - ctx.user.use_list = use_list + ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL ctx.user.encoding = enc ctx.user.unicode_errors = err @@ -277,15 +273,12 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, def unpack(object stream, object object_hook=None, object list_hook=None, - use_list=None, encoding=None, unicode_errors="strict", + bint use_list=1, encoding=None, unicode_errors="strict", ): """Unpack an object from `stream`. Raises `ValueError` when `stream` has extra bytes. """ - if use_list is None: - warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) - use_list = 0 return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors, @@ -356,15 +349,11 @@ cdef class Unpacker(object): free(self.buf) self.buf = NULL - def __init__(self, file_like=None, Py_ssize_t read_size=0, use_list=None, + def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1, object object_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, object object_pairs_hook=None, ): - if use_list is None: - warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) - use_list = 0 - self.file_like = file_like if file_like: self.file_like_read = file_like.read From ac403ef68da911ac4735407ada88db02015bd520 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 02:45:37 +0900 Subject: [PATCH 0574/1172] Start 0.2.3dev --- msgpack/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index f343b7a..9bffd02 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 2, 2) +version = (0, 2, 3, 'dev1') From 927d29131dc8d2a9f606cf7c881606d47ace557b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 02:55:50 +0900 Subject: [PATCH 0575/1172] Write about warning in changelog. --- ChangeLog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 46b83ee..f49b577 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -4,6 +4,7 @@ Changes ------- +* Warn when use_list is not specified. It's default value will be changed in 0.3. Bugs fixed ----------- From 477d3b152f5d36a48a8083b3720def2dd1f5d1a7 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 24 Sep 2012 03:08:13 +0900 Subject: [PATCH 0576/1172] Fix warnings. --- test/test_obj.py | 2 +- test/test_pack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_obj.py b/test/test_obj.py index 12a149f..881e627 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -29,7 +29,7 @@ def test_decode_hook(): def test_decode_pairs_hook(): packed = packb([3, {1: 2, 3: 4}]) prod_sum = 1 * 2 + 3 * 4 - unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l)) + unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l), use_list=1) eq_(unpacked[1], prod_sum) @raises(ValueError) diff --git a/test/test_pack.py b/test/test_pack.py index 9009d35..6af87fd 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -112,7 +112,7 @@ def test_odict(): assert_equal(unpackb(packb(od), use_list=1), dict(seq)) def pair_hook(seq): return seq - assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook), seq) + assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1), seq) if __name__ == '__main__': From d56e2b2c8aa1005fbac3b584cd003ba0cdece2e2 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Tue, 25 Sep 2012 00:30:15 +1000 Subject: [PATCH 0577/1172] Use C++ function templating for skip()/construct() --- msgpack/_msgpack.pyx | 23 +++++++++++------------ msgpack/unpack.h | 1 + msgpack/unpack_template.h | 21 +++++++++++++-------- setup.py | 4 ++-- 4 files changed, 27 insertions(+), 22 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index e0a1043..0fc3739 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -208,8 +208,10 @@ cdef extern from "unpack.h": unsigned int ct PyObject* key - int template_execute(template_context* ctx, const_char_ptr data, - size_t len, size_t* off, bint construct) except -1 + ctypedef int (*execute_fn)(template_context* ctx, const_char_ptr data, + size_t len, size_t* off) except -1 + execute_fn template_construct + execute_fn template_skip void template_init(template_context* ctx) object template_data(template_context* ctx) @@ -257,7 +259,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, if not PyCallable_Check(list_hook): raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook - ret = template_execute(&ctx, buf, buf_len, &off, 1) + ret = template_construct(&ctx, buf, buf_len, &off) if ret == 1: obj = template_data(&ctx) if off < buf_len: @@ -455,16 +457,13 @@ cdef class Unpacker(object): else: self.file_like = None - cdef object _unpack(self, bint construct): + cdef object _unpack(self, execute_fn execute): cdef int ret cdef object obj while 1: - ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head, construct) + ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) if ret == 1: - if construct: - obj = template_data(&self.ctx) - else: - obj = None + obj = template_data(&self.ctx) template_init(&self.ctx) return obj elif ret == 0: @@ -477,17 +476,17 @@ cdef class Unpacker(object): def unpack(self): """unpack one object""" - return self._unpack(1) + return self._unpack(template_construct) def skip(self): """read and ignore one object, returning None""" - return self._unpack(0) + return self._unpack(template_skip) def __iter__(self): return self def __next__(self): - return self._unpack(1) + return self._unpack(template_construct) # for debug. #def _buf(self): diff --git a/msgpack/unpack.h b/msgpack/unpack.h index a106f9c..3c9d4be 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -41,6 +41,7 @@ typedef struct unpack_user { #define msgpack_unpack_user unpack_user +typedef int (*execute_fn)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off); struct template_context; typedef struct template_context template_context; diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 5495a51..e0cf42e 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -95,7 +95,8 @@ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context } -msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off, int construct) +template +msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) { assert(len >= *off); @@ -380,6 +381,8 @@ _header_again: _finish: + if (!construct) + msgpack_unpack_callback(_nil)(user, &obj); stack[0].obj = obj; ++p; ret = 1; @@ -405,13 +408,6 @@ _end: #undef construct_cb } - -#undef msgpack_unpack_func -#undef msgpack_unpack_callback -#undef msgpack_unpack_struct -#undef msgpack_unpack_object -#undef msgpack_unpack_user - #undef push_simple_value #undef push_fixed_value #undef push_variable_value @@ -419,6 +415,15 @@ _end: #undef again_fixed_trail_if_zero #undef start_container +static const execute_fn template_construct = &template_execute; +static const execute_fn template_skip = &template_execute; + +#undef msgpack_unpack_func +#undef msgpack_unpack_callback +#undef msgpack_unpack_struct +#undef msgpack_unpack_object +#undef msgpack_unpack_user + #undef NEXT_CS /* vim: set ts=4 sw=4 noexpandtab */ diff --git a/setup.py b/setup.py index 86b0b34..708fa13 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ except ImportError: def cythonize(src): sys.stderr.write("cythonize: %r\n" % (src,)) - cython_compiler.compile([src]) + cython_compiler.compile([src], cplus=True) def ensure_source(src): pyx = os.path.splitext(src)[0] + '.pyx' @@ -67,7 +67,7 @@ if have_cython: else: Sdist = sdist -sources = ['msgpack/_msgpack.c'] +sources = ['msgpack/_msgpack.cpp'] libraries = [] if sys.platform == 'win32': libraries.append('ws2_32') From 0431a766f4e069d74627441aa3facbc7e64e4511 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Tue, 25 Sep 2012 01:18:33 +1000 Subject: [PATCH 0578/1172] read_array/map_header functionality --- msgpack/_msgpack.pyx | 10 ++++++ msgpack/unpack_template.h | 63 +++++++++++++++++++++++++++++++++++++ test/test_read_size.py | 66 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 test/test_read_size.py diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 0fc3739..7131d1f 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -212,6 +212,8 @@ cdef extern from "unpack.h": size_t len, size_t* off) except -1 execute_fn template_construct execute_fn template_skip + execute_fn read_array_header + execute_fn read_map_header void template_init(template_context* ctx) object template_data(template_context* ctx) @@ -482,6 +484,14 @@ cdef class Unpacker(object): """read and ignore one object, returning None""" return self._unpack(template_skip) + def read_array_header(self): + """assuming the next object is an array, return its size n, such that the next n unpack() calls will iterate over its contents.""" + return self._unpack(read_array_header) + + def read_map_header(self): + """assuming the next object is a map, return its size n, such that the next n * 2 unpack() calls will iterate over its key-value pairs.""" + return self._unpack(read_map_header) + def __iter__(self): return self diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index e0cf42e..69ef6e2 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -408,6 +408,10 @@ _end: #undef construct_cb } +#undef SWITCH_RANGE_BEGIN +#undef SWITCH_RANGE +#undef SWITCH_RANGE_DEFAULT +#undef SWITCH_RANGE_END #undef push_simple_value #undef push_fixed_value #undef push_variable_value @@ -415,8 +419,67 @@ _end: #undef again_fixed_trail_if_zero #undef start_container +template +msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) +{ + assert(len >= *off); + uint32_t size; + const unsigned char *const p = (unsigned char*)data + *off; + +#define inc_offset(inc) \ + if (len - *off < inc) \ + return 0; \ + *off += inc; + + switch (*p) { + case var_offset: + inc_offset(3); + size = _msgpack_load16(uint16_t, p + 1); + break; + case var_offset + 1: + inc_offset(5); + size = _msgpack_load32(uint32_t, p + 1); + break; +#ifdef USE_CASE_RANGE + case fixed_offset + 0x0 ... fixed_offset + 0xf: +#else + case fixed_offset + 0x0: + case fixed_offset + 0x1: + case fixed_offset + 0x2: + case fixed_offset + 0x3: + case fixed_offset + 0x4: + case fixed_offset + 0x5: + case fixed_offset + 0x6: + case fixed_offset + 0x7: + case fixed_offset + 0x8: + case fixed_offset + 0x9: + case fixed_offset + 0xa: + case fixed_offset + 0xb: + case fixed_offset + 0xc: + case fixed_offset + 0xd: + case fixed_offset + 0xe: + case fixed_offset + 0xf: +#endif + ++*off; + size = ((unsigned int)*p) & 0x0f; + break; + default: + PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); + return -1; + } + msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj); + return 1; +} + +#undef SWITCH_RANGE_BEGIN +#undef SWITCH_RANGE +#undef SWITCH_RANGE_DEFAULT +#undef SWITCH_RANGE_END + static const execute_fn template_construct = &template_execute; static const execute_fn template_skip = &template_execute; +static const execute_fn read_array_header = &template_container_header<0x90, 0xdc>; +static const execute_fn read_map_header = &template_container_header<0x80, 0xde>; #undef msgpack_unpack_func #undef msgpack_unpack_callback diff --git a/test/test_read_size.py b/test/test_read_size.py new file mode 100644 index 0000000..714f963 --- /dev/null +++ b/test/test_read_size.py @@ -0,0 +1,66 @@ +"""Test Unpacker's read_array_header and read_map_header methods""" +from msgpack import packb, Unpacker +UnexpectedTypeException = ValueError + +def test_read_array_header(): + unpacker = Unpacker() + unpacker.feed(packb(['a', 'b', 'c'])) + assert unpacker.read_array_header() == 3 + assert unpacker.unpack() == 'a' + assert unpacker.unpack() == 'b' + assert unpacker.unpack() == 'c' + try: + unpacker.unpack() + assert 0, 'should raise exception' + except StopIteration: + assert 1, 'okay' + + +def test_read_map_header(): + unpacker = Unpacker() + unpacker.feed(packb({'a': 'A'})) + assert unpacker.read_map_header() == 1 + assert unpacker.unpack() == 'a' + assert unpacker.unpack() == 'A' + try: + unpacker.unpack() + assert 0, 'should raise exception' + except StopIteration: + assert 1, 'okay' + +def test_incorrect_type_array(): + unpacker = Unpacker() + unpacker.feed(packb(1)) + try: + unpacker.read_array_header() + assert 0, 'should raise exception' + except UnexpectedTypeException: + assert 1, 'okay' + +def test_incorrect_type_map(): + unpacker = Unpacker() + unpacker.feed(packb(1)) + try: + unpacker.read_map_header() + assert 0, 'should raise exception' + except UnexpectedTypeException: + assert 1, 'okay' + +def test_correct_type_nested_array(): + unpacker = Unpacker() + unpacker.feed(packb({'a': ['b', 'c', 'd']})) + try: + unpacker.read_array_header() + assert 0, 'should raise exception' + except UnexpectedTypeException: + assert 1, 'okay' + +def test_incorrect_type_nested_map(): + unpacker = Unpacker() + unpacker.feed(packb([{'a': 'b'}])) + try: + unpacker.read_map_header() + assert 0, 'should raise exception' + except UnexpectedTypeException: + assert 1, 'okay' + From 9d9c3eecb846c6a927a31aae394dea39fa75aef4 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Sun, 23 Sep 2012 17:26:16 +1000 Subject: [PATCH 0579/1172] Packer.pack_array/map_header to correspond to read functions --- msgpack/_msgpack.pyx | 11 +++++++++++ test/test_pack.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 7131d1f..18a75ca 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -178,6 +178,17 @@ cdef class Packer(object): self.pk.length = 0 return buf + cpdef pack_array_header(self, size_t size): + msgpack_pack_array(&self.pk, size) + buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) + self.pk.length = 0 + return buf + + cpdef pack_map_header(self, size_t size): + msgpack_pack_map(&self.pk, size) + buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) + self.pk.length = 0 + return buf def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'): """ diff --git a/test/test_pack.py b/test/test_pack.py index b216c46..937141d 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -91,6 +91,35 @@ def testPackFloat(): assert_equal(packb(1.0, use_single_float=True), b'\xca' + struct.pack('>f', 1.0)) assert_equal(packb(1.0, use_single_float=False), b'\xcb' + struct.pack('>d', 1.0)) +def testArraySize(sizes=[0, 5, 50, 1000]): + bio = six.BytesIO() + packer = Packer() + for size in sizes: + bio.write(packer.pack_array_header(size)) + for i in range(size): + bio.write(packer.pack(i)) + + bio.seek(0) + unpacker = Unpacker(bio) + for size in sizes: + assert unpacker.unpack() == tuple(range(size)) + +def testMapSize(sizes=[0, 5, 50, 1000]): + bio = six.BytesIO() + packer = Packer() + for size in sizes: + bio.write(packer.pack_map_header(size)) + for i in range(size): + bio.write(packer.pack(i)) # key + bio.write(packer.pack(i * 2)) # value + + bio.seek(0) + unpacker = Unpacker(bio) + for size in sizes: + assert unpacker.unpack() == {i: i * 2 for i in range(size)} + + + class odict(dict): '''Reimplement OrderedDict to run test on Python 2.6''' From d5f99959cc2ec393c13fc9e44714351272bac7fc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 1 Oct 2012 01:34:58 +0900 Subject: [PATCH 0580/1172] Fix some test failure. --- test/test_pack.py | 6 +++--- test/test_read_size.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/test_pack.py b/test/test_pack.py index ff1eeef..21c2bd7 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -99,9 +99,9 @@ def testArraySize(sizes=[0, 5, 50, 1000]): bio.write(packer.pack(i)) bio.seek(0) - unpacker = Unpacker(bio) + unpacker = Unpacker(bio, use_list=1) for size in sizes: - assert unpacker.unpack() == tuple(range(size)) + assert unpacker.unpack() == list(range(size)) def testMapSize(sizes=[0, 5, 50, 1000]): bio = six.BytesIO() @@ -115,7 +115,7 @@ def testMapSize(sizes=[0, 5, 50, 1000]): bio.seek(0) unpacker = Unpacker(bio) for size in sizes: - assert unpacker.unpack() == {i: i * 2 for i in range(size)} + assert unpacker.unpack() == dict((i, i * 2) for i in range(size)) diff --git a/test/test_read_size.py b/test/test_read_size.py index 714f963..e130805 100644 --- a/test/test_read_size.py +++ b/test/test_read_size.py @@ -6,9 +6,9 @@ def test_read_array_header(): unpacker = Unpacker() unpacker.feed(packb(['a', 'b', 'c'])) assert unpacker.read_array_header() == 3 - assert unpacker.unpack() == 'a' - assert unpacker.unpack() == 'b' - assert unpacker.unpack() == 'c' + assert unpacker.unpack() == b'a' + assert unpacker.unpack() == b'b' + assert unpacker.unpack() == b'c' try: unpacker.unpack() assert 0, 'should raise exception' @@ -20,8 +20,8 @@ def test_read_map_header(): unpacker = Unpacker() unpacker.feed(packb({'a': 'A'})) assert unpacker.read_map_header() == 1 - assert unpacker.unpack() == 'a' - assert unpacker.unpack() == 'A' + assert unpacker.unpack() == B'a' + assert unpacker.unpack() == B'A' try: unpacker.unpack() assert 0, 'should raise exception' From 87f292cbf929c0cf8c8de867a5e7dd285cf1550a Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Thu, 4 Oct 2012 11:26:29 +1000 Subject: [PATCH 0581/1172] Allow packed data to be captured while executing skip(), etc. --- msgpack/_msgpack.pyx | 37 +++++++++++++++++++++++++------------ test/test_unpack_raw.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 12 deletions(-) create mode 100644 test/test_unpack_raw.py diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 7fcfcb0..71f9810 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -467,11 +467,16 @@ cdef class Unpacker(object): else: self.file_like = None - cdef object _unpack(self, execute_fn execute): + cdef object _unpack(self, execute_fn execute, object write_bytes): cdef int ret cdef object obj + cdef size_t prev_head while 1: + prev_head = self.buf_head ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + if write_bytes is not None: + write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head)) + if ret == 1: obj = template_data(&self.ctx) template_init(&self.ctx) @@ -484,27 +489,35 @@ cdef class Unpacker(object): else: raise ValueError("Unpack failed: error = %d" % (ret,)) - def unpack(self): - """unpack one object""" - return self._unpack(template_construct) + def unpack(self, object write_bytes=None): + """ + unpack one object - def skip(self): - """read and ignore one object, returning None""" - return self._unpack(template_skip) + If write_bytes is not None, it will be called with parts of the raw message as it is unpacked. + """ + return self._unpack(template_construct, write_bytes) - def read_array_header(self): + def skip(self, object write_bytes=None): + """ + read and ignore one object, returning None + + If write_bytes is not None, it will be called with parts of the raw message as it is unpacked. + """ + return self._unpack(template_skip, write_bytes) + + def read_array_header(self, object write_bytes=None): """assuming the next object is an array, return its size n, such that the next n unpack() calls will iterate over its contents.""" - return self._unpack(read_array_header) + return self._unpack(read_array_header, write_bytes) - def read_map_header(self): + def read_map_header(self, object write_bytes=None): """assuming the next object is a map, return its size n, such that the next n * 2 unpack() calls will iterate over its key-value pairs.""" - return self._unpack(read_map_header) + return self._unpack(read_map_header, write_bytes) def __iter__(self): return self def __next__(self): - return self._unpack(template_construct) + return self._unpack(template_construct, None) # for debug. #def _buf(self): diff --git a/test/test_unpack_raw.py b/test/test_unpack_raw.py new file mode 100644 index 0000000..ed2e04b --- /dev/null +++ b/test/test_unpack_raw.py @@ -0,0 +1,32 @@ +"""Tests for cases where the user seeks to obtain packed msgpack objects""" + +from nose import main +from nose.tools import * +import six +from msgpack import Unpacker, packb + +def test_write_bytes(): + unpacker = Unpacker() + unpacker.feed(b'abc') + f = six.BytesIO() + assert_equal(unpacker.unpack(f.write), ord('a')) + assert_equal(f.getvalue(), b'a') + f.truncate(0) + assert_is_none(unpacker.skip(f.write)) + assert_equal(f.getvalue(), b'b') + f.truncate(0) + assert_is_none(unpacker.skip()) + assert_equal(f.getvalue(), b'') + +def test_write_bytes_multi_buffer(): + long_val = (5) * 100 + expected = packb(long_val) + unpacker = Unpacker(six.BytesIO(expected), read_size=3, max_buffer_size=3) + + f = six.BytesIO() + unpacked = unpacker.unpack(f.write) + assert_equal(unpacked, long_val) + assert_equal(f.getvalue(), expected) + +if __name__ == '__main__': + main() From 89ce16df39f67ac77785a63b4111c353f0a606a3 Mon Sep 17 00:00:00 2001 From: Alexei Romanoff Date: Fri, 12 Oct 2012 12:32:32 +0300 Subject: [PATCH 0582/1172] A segfault fixed in the issue https://github.com/msgpack/msgpack-python/issues/28 --- msgpack/unpack.h | 3 +++ msgpack/unpack_template.h | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 5ec7dbc..61e5d91 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -207,6 +207,9 @@ static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_objec { if (u->object_hook) { PyObject *new_c = PyEval_CallFunction(u->object_hook, "(O)", *c); + if (!new_c) + return -1; + Py_DECREF(*c); *c = new_c; } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 7d07601..9d13062 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -361,7 +361,7 @@ _push: if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } if(++c->count == c->size) { obj = c->obj; - construct_cb(_map_end)(user, &obj); + if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; } --top; /*printf("stack pop %d\n", top);*/ goto _push; From 4ea952f39dc5ff1231f780252d8d4efb16b2be3b Mon Sep 17 00:00:00 2001 From: Alexei Romanoff Date: Fri, 12 Oct 2012 12:34:18 +0300 Subject: [PATCH 0583/1172] Added unit-test for issue https://github.com/msgpack/msgpack-python/issues/28 --- test/test_obj.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/test_obj.py b/test/test_obj.py index 881e627..15e192c 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -49,5 +49,18 @@ def test_array_hook(): unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1) eq_(unpacked, '123') + +class DecodeError(Exception): + pass + +def bad_complex_decoder(o): + raise DecodeError("Ooops!") + + +@raises(DecodeError) +def test_an_exception_in_objecthook1(): + packed = packb({1: {'__complex__': True, 'real': 1, 'imag': 2}}) + unpackb(packed, object_hook=bad_complex_decoder) + if __name__ == '__main__': main() From cf89f18be7614d6d55bb9eb7e9bf0e10d42a8508 Mon Sep 17 00:00:00 2001 From: Alexei Romanoff Date: Fri, 12 Oct 2012 13:19:53 +0300 Subject: [PATCH 0584/1172] segfault fixed when data is unpacked using `list_hook`, this bug is a twin to #28. Unit-test is also attached. --- msgpack/unpack.h | 2 ++ msgpack/unpack_template.h | 2 +- test/test_obj.py | 8 ++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 61e5d91..3dc88e5 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -163,6 +163,8 @@ static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_obj { if (u->list_hook) { PyObject *new_c = PyEval_CallFunction(u->list_hook, "(O)", *c); + if (!new_c) + return -1; Py_DECREF(*c); *c = new_c; } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 9d13062..8a57f0d 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -347,7 +347,7 @@ _push: if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } if(++c->count == c->size) { obj = c->obj; - construct_cb(_array_end)(user, &obj); + if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; } --top; /*printf("stack pop %d\n", top);*/ goto _push; diff --git a/test/test_obj.py b/test/test_obj.py index 15e192c..1d9024b 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -62,5 +62,13 @@ def test_an_exception_in_objecthook1(): packed = packb({1: {'__complex__': True, 'real': 1, 'imag': 2}}) unpackb(packed, object_hook=bad_complex_decoder) + +@raises(DecodeError) +def test_an_exception_in_objecthook2(): + packed = packb({1: [{'__complex__': True, 'real': 1, 'imag': 2}]}) + unpackb(packed, list_hook=bad_complex_decoder, use_list=1) + + + if __name__ == '__main__': main() From 541d22d00434863ce1e3607950c199edbd2b850a Mon Sep 17 00:00:00 2001 From: Alexei Romanoff Date: Fri, 12 Oct 2012 13:32:29 +0300 Subject: [PATCH 0585/1172] Added example of using default/object_hook into README --- README.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README.rst b/README.rst index 1da6ece..ee896ce 100644 --- a/README.rst +++ b/README.rst @@ -61,6 +61,36 @@ stream. for unpacked in unpacker: print unpacked +packing/unpacking of custom data type +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Also possible to pack/unpack user's data types. Here is an example for +``datetime.datetime``. + +:: + import datetime + + import msgpack + + useful_dict = { + "id": 1, + "created": datetime.datetime.now(), + } + + def decode_datetime(obj): + if b'__datetime__' in obj: + obj = datetime.datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f") + return obj + + def encode_datetime(obj): + if isinstance(obj, datetime.datetime): + return {'__datetime__': True, 'as_str': obj.strftime("%Y%m%dT%H:%M:%S.%f")} + return obj + + + packed_dict = msgpack.packb(useful_dict, default=encode_datetime) + this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime) + INSTALL --------- From fa1c4745ec4c8cf8bad5f35be8ddb7fd9e28532e Mon Sep 17 00:00:00 2001 From: Alexei Romanoff Date: Fri, 12 Oct 2012 15:25:14 +0300 Subject: [PATCH 0586/1172] README formatting has been improved --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index ee896ce..834c86c 100644 --- a/README.rst +++ b/README.rst @@ -68,6 +68,7 @@ Also possible to pack/unpack user's data types. Here is an example for ``datetime.datetime``. :: + import datetime import msgpack From 62e8f400244a449ab4ca72991ae8f06610298dc6 Mon Sep 17 00:00:00 2001 From: Spiros Eliopoulos Date: Thu, 1 Nov 2012 00:41:06 -0400 Subject: [PATCH 0587/1172] Fix typo in README seed -> feed --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 834c86c..bcfd98c 100644 --- a/README.rst +++ b/README.rst @@ -57,7 +57,7 @@ stream. data = buf.read(4) if not data: break - unpacker.seed(buf.read(16)) + unpacker.feed(buf.read(16)) for unpacked in unpacker: print unpacked From 30233a5a995e23b72319598d11ebba084497a18b Mon Sep 17 00:00:00 2001 From: Spiros Eliopoulos Date: Thu, 1 Nov 2012 00:55:33 -0400 Subject: [PATCH 0588/1172] Fix Unpacker example in README The example did not properly deserialize, since it was dropping bytes from the input stream. --- README.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index bcfd98c..c15d46b 100644 --- a/README.rst +++ b/README.rst @@ -52,14 +52,15 @@ stream. buf.seek(0) - unpacker = msgpack.Unpacker() - while True: - data = buf.read(4) + unpacker = msgpack.Unpacker() + while True: + data = buf.read(16) if not data: break - unpacker.feed(buf.read(16)) - for unpacked in unpacker: - print unpacked + unpacker.feed(data) + + for unpacked in unpacker: + print unpacked packing/unpacking of custom data type ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From d025d908821f60423fd4ad20f04500f3a289783a Mon Sep 17 00:00:00 2001 From: Spiros Eliopoulos Date: Thu, 1 Nov 2012 11:28:11 -0400 Subject: [PATCH 0589/1172] Put Unpacker read loop back into buffer read loop So it works for streaming as intended. --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index c15d46b..dcd3eb1 100644 --- a/README.rst +++ b/README.rst @@ -52,15 +52,15 @@ stream. buf.seek(0) - unpacker = msgpack.Unpacker() - while True: + unpacker = msgpack.Unpacker() + while True: data = buf.read(16) if not data: break unpacker.feed(data) - for unpacked in unpacker: - print unpacked + for unpacked in unpacker: + print unpacked packing/unpacking of custom data type ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 2f0078d3955ad151452b92d570661d1b5779d7b6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 6 Nov 2012 02:10:36 +0900 Subject: [PATCH 0590/1172] Add travis config. --- .travis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..593d61e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +language: python +python: + - 2.5 + - 2.6 + - 2.7 + - 3.2 + - 3.3 + +install: "pip install cython --use-mirrors" +script: nosetests From 3ec2e6e729ee71d0284140d3f494529788cbcfd5 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 6 Nov 2012 02:32:59 +0900 Subject: [PATCH 0591/1172] Add six to .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 593d61e..7286637 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,5 +6,5 @@ python: - 3.2 - 3.3 -install: "pip install cython --use-mirrors" +install: "pip install cython six --use-mirrors" script: nosetests From dbf50c9f789b831e0e8ec582a9947708175b5d5e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 6 Nov 2012 09:35:06 +0900 Subject: [PATCH 0592/1172] Add travis status. --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index dcd3eb1..8cebc01 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,9 @@ MessagePack Python Binding :version: 0.2.0 :date: 2012-06-27 +.. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png + :target: https://travis-ci.org/#!/msgpack/msgpack-python + HOW TO USE ----------- From c75ef976d7230ceee884e24127736b6771351d80 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 6 Nov 2012 09:37:55 +0900 Subject: [PATCH 0593/1172] Fix build error on first time. --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5be92bf..b9af8c3 100644 --- a/setup.py +++ b/setup.py @@ -34,8 +34,7 @@ Install Cython >= 0.16 or install msgpack from PyPI. os.stat(src).st_mtime < os.stat(pyx).st_mtime and have_cython): cythonize(pyx) - else: - return src + return src class BuildExt(build_ext): From eb61b4de9e052e4c126b21348f83695bcb30534a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 6 Nov 2012 09:46:02 +0900 Subject: [PATCH 0594/1172] (travis) Install msgpack-python itself. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7286637..4701750 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,5 +6,5 @@ python: - 3.2 - 3.3 -install: "pip install cython six --use-mirrors" +install: "pip install cython six . --use-mirrors" script: nosetests From e404c9845f5eab127e4d0c66b7ddf7cd9733c22b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 6 Nov 2012 10:39:10 +0900 Subject: [PATCH 0595/1172] (travis) Install itself. --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4701750..6bf4336 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,5 +6,8 @@ python: - 3.2 - 3.3 -install: "pip install cython six . --use-mirrors" +install: + - "pip install cython six --use-mirrors" + - "python setup.py install" + script: nosetests From 0ef52869e39b597a6937f46fe00147227034ce02 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 7 Nov 2012 02:00:08 +0900 Subject: [PATCH 0596/1172] Fix unpack error on Python 3.2. ctx.user.encoding and ctx.user.unicode_errors may refer to deallocated string. --- msgpack/_msgpack.pyx | 58 ++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 7fcfcb0..f665306 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -1,8 +1,6 @@ # coding: utf-8 #cython: embedsignature=True -import warnings - from cpython cimport * cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" @@ -233,7 +231,9 @@ cdef extern from "unpack.h": void template_init(template_context* ctx) object template_data(template_context* ctx) -cdef inline init_ctx(template_context *ctx, object object_hook, object object_pairs_hook, object list_hook, bint use_list, encoding, unicode_errors): +cdef inline init_ctx(template_context *ctx, + object object_hook, object object_pairs_hook, object list_hook, + bint use_list, char* encoding, char* unicode_errors): template_init(ctx) ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL @@ -259,20 +259,8 @@ cdef inline init_ctx(template_context *ctx, object object_hook, object object_pa raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook - if encoding is None: - ctx.user.encoding = NULL - ctx.user.unicode_errors = NULL - else: - if isinstance(encoding, unicode): - _bencoding = encoding.encode('ascii') - else: - _bencoding = encoding - ctx.user.encoding = PyBytes_AsString(_bencoding) - if isinstance(unicode_errors, unicode): - _berrors = unicode_errors.encode('ascii') - else: - _berrors = unicode_errors - ctx.user.unicode_errors = PyBytes_AsString(_berrors) + ctx.user.encoding = encoding + ctx.user.unicode_errors = unicode_errors def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=1, encoding=None, unicode_errors="strict", @@ -288,10 +276,22 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef char* buf cdef Py_ssize_t buf_len + cdef char* cenc = NULL + cdef char* cerr = NULL PyObject_AsReadBuffer(packed, &buf, &buf_len) - init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors) + if encoding is not None: + if isinstance(encoding, unicode): + encoding = encoding.encode('ascii') + cenc = PyBytes_AsString(encoding) + + if unicode_errors is not None: + if isinstance(unicode_errors, unicode): + unicode_errors = unicode_errors.encode('ascii') + cerr = PyBytes_AsString(unicode_errors) + + init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) ret = template_construct(&ctx, buf, buf_len, &off) if ret == 1: obj = template_data(&ctx) @@ -370,10 +370,7 @@ cdef class Unpacker(object): cdef object file_like_read cdef Py_ssize_t read_size cdef object object_hook - cdef object _bencoding - cdef object _berrors - cdef char *encoding - cdef char *unicode_errors + cdef object encoding, unicode_errors cdef size_t max_buffer_size def __cinit__(self): @@ -387,6 +384,8 @@ cdef class Unpacker(object): object object_hook=None, object object_pairs_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, ): + cdef char *cenc, *cerr + self.file_like = file_like if file_like: self.file_like_read = file_like.read @@ -406,7 +405,20 @@ cdef class Unpacker(object): self.buf_size = read_size self.buf_head = 0 self.buf_tail = 0 - init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors) + + if encoding is not None: + if isinstance(encoding, unicode): + encoding = encoding.encode('ascii') + self.encoding = encoding + cenc = PyBytes_AsString(encoding) + + if unicode_errors is not None: + if isinstance(unicode_errors, unicode): + unicode_errors = unicode_errors.encode('ascii') + self.unicode_errors = unicode_errors + cerr = PyBytes_AsString(unicode_errors) + + init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) def feed(self, object next_bytes): cdef char* buf From df6b969a8d40fc1b2f934801a1088eda24b73ab2 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 7 Nov 2012 02:04:42 +0900 Subject: [PATCH 0597/1172] (travis) Fix test script --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6bf4336..bbd5ff1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,4 +10,4 @@ install: - "pip install cython six --use-mirrors" - "python setup.py install" -script: nosetests +script: "nosetests -w test" From ec655b9f2ccd4a0fb6e9fc9c9f1d8b88ace69e72 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 7 Nov 2012 02:23:57 +0900 Subject: [PATCH 0598/1172] Fix segmentation fault. --- msgpack/_msgpack.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index f665306..9e79910 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -384,7 +384,7 @@ cdef class Unpacker(object): object object_hook=None, object object_pairs_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, ): - cdef char *cenc, *cerr + cdef char *cenc=NULL, *cerr=NULL self.file_like = file_like if file_like: From 67d8cc6c4f0e657674175fadda702b7bf95937c3 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 7 Nov 2012 02:24:26 +0900 Subject: [PATCH 0599/1172] (travis) Python 2.5 can't pass tests. Python 2.5 doesn't have b'' literal. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bbd5ff1..9247c78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - 2.5 - 2.6 - 2.7 - 3.2 From b14caa419c9ac35779b3212d367c24f36f40e484 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 7 Nov 2012 08:15:46 +0900 Subject: [PATCH 0600/1172] Use system cython. --- .travis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9247c78..2f3171c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,10 @@ python: - 3.3 install: - - "pip install cython six --use-mirrors" - - "python setup.py install" + - sudo apt-get update -qq + - sudo apt-get install -q cython + - cython --cplus msgpack/_msgpack.pyx + - pip install six --use-mirrors + - python setup.py install script: "nosetests -w test" From 1c389135b88d3fc8be4151228fff1fb5764f119b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 7 Nov 2012 08:28:30 +0900 Subject: [PATCH 0601/1172] (travis) Travis doesn't support Python 3.3 yet. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2f3171c..11835cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ python: - 2.6 - 2.7 - 3.2 - - 3.3 install: - sudo apt-get update -qq From 6b782232317b3e00f8896fb36567f8ad8c3c3345 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 6 Dec 2012 19:15:04 +0900 Subject: [PATCH 0602/1172] Fix test failuar. --- test/test_unpack_raw.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/test_unpack_raw.py b/test/test_unpack_raw.py index ed2e04b..15d9c93 100644 --- a/test/test_unpack_raw.py +++ b/test/test_unpack_raw.py @@ -5,19 +5,21 @@ from nose.tools import * import six from msgpack import Unpacker, packb + def test_write_bytes(): unpacker = Unpacker() unpacker.feed(b'abc') f = six.BytesIO() assert_equal(unpacker.unpack(f.write), ord('a')) assert_equal(f.getvalue(), b'a') - f.truncate(0) - assert_is_none(unpacker.skip(f.write)) + f = six.BytesIO() + assert unpacker.skip(f.write) is None assert_equal(f.getvalue(), b'b') - f.truncate(0) - assert_is_none(unpacker.skip()) + f = six.BytesIO() + assert unpacker.skip() is None assert_equal(f.getvalue(), b'') + def test_write_bytes_multi_buffer(): long_val = (5) * 100 expected = packb(long_val) From caecc0098eeb631bb0aeca1fdba397ddd96ca329 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Thu, 6 Dec 2012 23:01:12 +1100 Subject: [PATCH 0603/1172] Change Unpacker example to read from stream --- README.rst | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 8cebc01..8154098 100644 --- a/README.rst +++ b/README.rst @@ -55,15 +55,9 @@ stream. buf.seek(0) - unpacker = msgpack.Unpacker() - while True: - data = buf.read(16) - if not data: - break - unpacker.feed(data) - - for unpacked in unpacker: - print unpacked + unpacker = msgpack.Unpacker(buf) + for unpacked in unpacker: + print unpacked packing/unpacking of custom data type ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From c567ad1c5220a9da145d41b4e61f3a411d32ce57 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Thu, 6 Dec 2012 23:10:25 +1100 Subject: [PATCH 0604/1172] Describe object_pairs_hook in README --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 8154098..f3779f1 100644 --- a/README.rst +++ b/README.rst @@ -90,6 +90,9 @@ Also possible to pack/unpack user's data types. Here is an example for packed_dict = msgpack.packb(useful_dict, default=encode_datetime) this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime) +``Unpacker``'s ``object_hook`` callback receives a dict; the +``object_pairs_hook`` callback may instead be used to receive a list of +key-value pairs. INSTALL --------- From 53b67f14496d71b3aa896d3311367401dc3c00a3 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 6 Dec 2012 21:17:44 +0900 Subject: [PATCH 0605/1172] Add changelog --- ChangeLog.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 6a4d27b..43ee723 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -5,7 +5,8 @@ Changes ------- * Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman) - +* Add capturing feature. You can pass the writable object to + ``Unpacker.unpack()`` as a second parameter. 0.2.3 ======= From de3724c1de8b0320e6cab3736404ab9857b7a951 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Thu, 6 Dec 2012 23:34:18 +1100 Subject: [PATCH 0606/1172] README documentation of advanced Unpacker features --- README.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README.rst b/README.rst index f3779f1..3f5e823 100644 --- a/README.rst +++ b/README.rst @@ -94,6 +94,36 @@ Also possible to pack/unpack user's data types. Here is an example for ``object_pairs_hook`` callback may instead be used to receive a list of key-value pairs. + +advanced unpacking control +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As an alternative to iteration, ``Unpacker`` objects provide ``unpack``, +``skip``, ``read_array_header`` and ``read_map_header`` methods. The former two +read an entire message from the stream, respectively deserialising and returning +the result, or ignoring it. The latter two methods return the number of elements +in the upcoming container, so that each element in an array, or key-value pair +in a map, can be unpacked or skipped individually. + +Each of these methods may optionally write the packed data it reads to a +callback function: + +:: + + from io import BytesIO + + def distribute(unpacker, get_worker): + nelems = unpacker.read_map_header() + for i in range(nelems): + # Select a worker for the given key + key = unpacker.unpack() + worker = get_worker(key) + + # Send the value as a packed message to worker + bytestream = BytesIO() + unpacker.skip(bytestream.write) + worker.send(bytestream.getvalue()) + INSTALL --------- You can use ``pip`` or ``easy_install`` to install msgpack:: From 6d4115f64bb180f34161b8a517c9c2165af81deb Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Thu, 6 Dec 2012 23:36:16 +1100 Subject: [PATCH 0607/1172] Minor grammar fixes --- README.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 3f5e823..9a437eb 100644 --- a/README.rst +++ b/README.rst @@ -35,13 +35,13 @@ To unpack it to list, Use ``use_list`` option. >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=True) [1, 2, 3] -Read docstring for other options. +Read the docstring for other options. streaming unpacking ^^^^^^^^^^^^^^^^^^^ -``Unpacker`` is "streaming unpacker". It unpacks multiple objects from one +``Unpacker`` is a "streaming unpacker". It unpacks multiple objects from one stream. :: @@ -59,10 +59,11 @@ stream. for unpacked in unpacker: print unpacked + packing/unpacking of custom data type ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Also possible to pack/unpack user's data types. Here is an example for +It is also possible to pack/unpack custom data types. Here is an example for ``datetime.datetime``. :: From 659d0961a319bbaabf994a972b06022c99e4465c Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Thu, 6 Dec 2012 23:36:46 +1100 Subject: [PATCH 0608/1172] Brief mention of Unpacker.feed in README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 9a437eb..1f5b1a2 100644 --- a/README.rst +++ b/README.rst @@ -42,7 +42,7 @@ streaming unpacking ^^^^^^^^^^^^^^^^^^^ ``Unpacker`` is a "streaming unpacker". It unpacks multiple objects from one -stream. +stream (or from bytes provided through its ``feed`` method). :: From fc41ed606d06becae4b53c8614b74768ae755cba Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Thu, 6 Dec 2012 23:44:27 +1100 Subject: [PATCH 0609/1172] Mention CPython and MessagePack in README --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index 1f5b1a2..5264ce9 100644 --- a/README.rst +++ b/README.rst @@ -9,6 +9,13 @@ MessagePack Python Binding .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png :target: https://travis-ci.org/#!/msgpack/msgpack-python +WHAT IT IS +---------- + +`MessagePack `_ is a fast, compact binary serialization format, suitable for +similar data to JSON. This package provides CPython bindings for reading and +writing MessagePack data. + HOW TO USE ----------- From 15f309c0b197abe86e94fb3840ff8979b35c7e0c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 6 Dec 2012 22:26:39 +0900 Subject: [PATCH 0610/1172] Add note about use_list. --- README.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.rst b/README.rst index 5264ce9..30a264d 100644 --- a/README.rst +++ b/README.rst @@ -42,6 +42,9 @@ To unpack it to list, Use ``use_list`` option. >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=True) [1, 2, 3] +The default behavior will be changed in the future. (probably 0.4) +You should always pass the ``use_list`` keyword argument. + Read the docstring for other options. @@ -151,6 +154,27 @@ or Windows SDK on Windows. (NOTE: Visual C++ Express 2010 doesn't support amd64. Windows SDK is recommanded way to build amd64 msgpack without any fee.) +PERFORMANCE NOTE +----------------- + +GC +^^ + +CPython's GC starts when growing allocated object. +This means unpacking may cause useless GC. +You can use ``gc.disable()`` when unpacking large message. + +use_list +^^^^^^^^^ +List is the default sequence type of Python. +But tuple is lighter than list. +You can use ``use_list=False`` while unpacking when performance is important. + +Python's dict can't use list as key and MessagePack allows array for key of mapping. +``use_list=False`` allows unpacking such message. +Another way to unpacking such object is using ``object_pairs_hook``. + + TEST ---- MessagePack uses `nosetest` for testing. From edd2e52373246c7d55ee59bf3bd46cda60e3be6d Mon Sep 17 00:00:00 2001 From: jnothman Date: Fri, 7 Dec 2012 00:53:17 +1100 Subject: [PATCH 0611/1172] Fix README re default use_list=True --- README.rst | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 30a264d..cdfb9e1 100644 --- a/README.rst +++ b/README.rst @@ -29,21 +29,20 @@ msgpack provides ``dumps`` and ``loads`` as alias for compatibility with ``pack`` and ``dump`` packs to file-like object. ``unpack`` and ``load`` unpacks from file-like object. +:: + >>> import msgpack >>> msgpack.packb([1, 2, 3]) '\x93\x01\x02\x03' >>> msgpack.unpackb(_) - (1, 2, 3) - - -``unpack`` unpacks msgpack's array to Python's tuple. -To unpack it to list, Use ``use_list`` option. - - >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=True) [1, 2, 3] -The default behavior will be changed in the future. (probably 0.4) -You should always pass the ``use_list`` keyword argument. +``unpack`` unpacks msgpack's array to Python's list, but can unpack to tuple:: + + >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False) + (1, 2, 3) + +You should always pass the ``use_list`` keyword argument. See performance issues relating to use_list_ below. Read the docstring for other options. From 56ef0d07debe19af355f284d54ee5f1dcf542592 Mon Sep 17 00:00:00 2001 From: jnothman Date: Fri, 7 Dec 2012 11:12:19 +1100 Subject: [PATCH 0612/1172] Warn about StopIteration in README --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 30a264d..61cc56c 100644 --- a/README.rst +++ b/README.rst @@ -116,6 +116,9 @@ the result, or ignoring it. The latter two methods return the number of elements in the upcoming container, so that each element in an array, or key-value pair in a map, can be unpacked or skipped individually. +Warning: these methods raise ``StopIteration`` when called at the end of the +stream. Unless caught, this may silently break an iteration. + Each of these methods may optionally write the packed data it reads to a callback function: From dd5c76b9552e371503535ce10a0314151e62fa28 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 7 Dec 2012 11:35:16 +0900 Subject: [PATCH 0613/1172] Add NOTE for changing default value of use_list. --- ChangeLog.rst | 6 ++++++ README.rst | 11 +++++++++-- msgpack/_msgpack.pyx | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 43ee723..279cb81 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -2,6 +2,12 @@ ===== :release date: in development +Inconpatible Changes +-------------------- + +* Default value of ``use_list`` is ``True`` for now. (It was ``False`` for 0.2.x) + You should pass it explicitly for compatibility to 0.2.x. + Changes ------- * Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman) diff --git a/README.rst b/README.rst index 58f6cb1..aa447f3 100644 --- a/README.rst +++ b/README.rst @@ -3,8 +3,8 @@ MessagePack Python Binding =========================== :author: INADA Naoki -:version: 0.2.0 -:date: 2012-06-27 +:version: 0.3.0 +:date: 2012-12-07 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png :target: https://travis-ci.org/#!/msgpack/msgpack-python @@ -16,6 +16,13 @@ WHAT IT IS similar data to JSON. This package provides CPython bindings for reading and writing MessagePack data. +NOTE for msgpack 0.2.x users +---------------------------- + +The default value of ``use_list`` keyword argument is ``True`` from 0.3.x. +You should pass the argument explicitly for backward compatibility. + + HOW TO USE ----------- diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 38940f0..5c202fc 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -327,7 +327,7 @@ cdef class Unpacker(object): `read_size` is used as `file_like.read(read_size)`. (default: min(1024**2, max_buffer_size)) - If `use_list` is true, msgpack list is deserialized to Python list. + If `use_list` is true (default), msgpack list is deserialized to Python list. Otherwise, it is deserialized to Python tuple. `object_hook` is same to simplejson. If it is not None, it should be callable From 219d47503ca982996b03f4396fdd2929c9905356 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 10 Dec 2012 00:31:19 +0900 Subject: [PATCH 0614/1172] Split exceptions. --- msgpack/_msgpack.pyx | 24 ++++++++++++++++-------- msgpack/exceptions.py | 23 +++++++++++++++++++++++ test/test_read_size.py | 6 +++--- test/test_sequnpack.py | 5 +++-- 4 files changed, 45 insertions(+), 13 deletions(-) create mode 100644 msgpack/exceptions.py diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 5c202fc..da70f0d 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -38,8 +38,13 @@ cdef extern from "pack.h": cdef int DEFAULT_RECURSE_LIMIT=511 -class BufferFull(Exception): - pass +from msgpack.exceptions import ( + UnpackException, + BufferFull, + OutOfData, + UnpackValueError, + ExtraData, + ) cdef class Packer(object): @@ -102,7 +107,7 @@ cdef class Packer(object): cdef dict d if nest_limit < 0: - raise ValueError("Too deep.") + raise UnpackValueError("recursion limit exceeded.") if o is None: ret = msgpack_pack_nil(&self.pk) @@ -296,7 +301,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, if ret == 1: obj = template_data(&ctx) if off < buf_len: - raise ValueError("Extra data.") + raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) return obj else: return None @@ -425,7 +430,7 @@ cdef class Unpacker(object): cdef Py_ssize_t buf_len if self.file_like is not None: raise AssertionError( - "unpacker.feed() is not be able to use with`file_like`.") + "unpacker.feed() is not be able to use with `file_like`.") PyObject_AsReadBuffer(next_bytes, &buf, &buf_len) self.append_buffer(buf, buf_len) @@ -479,7 +484,7 @@ cdef class Unpacker(object): else: self.file_like = None - cdef object _unpack(self, execute_fn execute, object write_bytes): + cdef object _unpack(self, execute_fn execute, object write_bytes, bint iter=0): cdef int ret cdef object obj cdef size_t prev_head @@ -497,7 +502,10 @@ cdef class Unpacker(object): if self.file_like is not None: self.read_from_file() continue - raise StopIteration("No more data to unpack.") + if iter: + raise StopIteration("No more data to unpack.") + else: + raise OutOfData("No more data to unpack.") else: raise ValueError("Unpack failed: error = %d" % (ret,)) @@ -539,7 +547,7 @@ cdef class Unpacker(object): return self def __next__(self): - return self._unpack(template_construct, None) + return self._unpack(template_construct, None, 1) # for debug. #def _buf(self): diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py new file mode 100644 index 0000000..0a75430 --- /dev/null +++ b/msgpack/exceptions.py @@ -0,0 +1,23 @@ +class UnpackException(Exception): + pass + + +class BufferFull(UnpackException): + pass + + +class OutOfData(UnpackException): + pass + + +class UnpackValueError(UnpackException, ValueError): + pass + + +class ExtraData(ValueError): + def __init__(self, unpacked, extra): + self.unpacked = unpacked + self.extra = extra + + def __str__(self): + return "unpack(b) recieved extra data." diff --git a/test/test_read_size.py b/test/test_read_size.py index e130805..4e6c2b9 100644 --- a/test/test_read_size.py +++ b/test/test_read_size.py @@ -1,5 +1,5 @@ """Test Unpacker's read_array_header and read_map_header methods""" -from msgpack import packb, Unpacker +from msgpack import packb, Unpacker, OutOfData UnexpectedTypeException = ValueError def test_read_array_header(): @@ -12,7 +12,7 @@ def test_read_array_header(): try: unpacker.unpack() assert 0, 'should raise exception' - except StopIteration: + except OutOfData: assert 1, 'okay' @@ -25,7 +25,7 @@ def test_read_map_header(): try: unpacker.unpack() assert 0, 'should raise exception' - except StopIteration: + except OutOfData: assert 1, 'okay' def test_incorrect_type_array(): diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 769f3ff..f767726 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -3,6 +3,7 @@ import six from msgpack import Unpacker, BufferFull +from msgpack.exceptions import OutOfData import nose def test_foobar(): @@ -17,7 +18,7 @@ def test_foobar(): try: o = unpacker.unpack() assert 0, "should raise exception" - except StopIteration: + except OutOfData: assert 1, "ok" unpacker.feed(b'foo') @@ -41,7 +42,7 @@ def test_foobar_skip(): try: o = unpacker.unpack() assert 0, "should raise exception" - except StopIteration: + except OutOfData: assert 1, "ok" def test_maxbuffersize(): From c7161e9403ce15c4bbea99cb482ea37a74311b45 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 10 Dec 2012 00:35:26 +0900 Subject: [PATCH 0615/1172] Add note for StopIteration => OutOfData. --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index aa447f3..270b8ff 100644 --- a/README.rst +++ b/README.rst @@ -19,9 +19,15 @@ writing MessagePack data. NOTE for msgpack 0.2.x users ---------------------------- +The msgpack 0.3 have some incompatible changes. + The default value of ``use_list`` keyword argument is ``True`` from 0.3.x. You should pass the argument explicitly for backward compatibility. +`Unpacker.unpack()` and some unpack methods now raises `OutOfData` +instead of `StopIteration`. +`StopIteration` is used for iterator protocol only. + HOW TO USE ----------- From 4480227e066e66fd771dc1702adf4929dede453d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 10 Dec 2012 00:39:04 +0900 Subject: [PATCH 0616/1172] Improve docstrings. --- msgpack/_msgpack.pyx | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index da70f0d..2c81cb4 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -426,6 +426,7 @@ cdef class Unpacker(object): init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) def feed(self, object next_bytes): + """Append `next_bytes` to internal buffer.""" cdef char* buf cdef Py_ssize_t buf_len if self.file_like is not None: @@ -523,7 +524,11 @@ cdef class Unpacker(object): """ unpack one object - If write_bytes is not None, it will be called with parts of the raw message as it is unpacked. + If write_bytes is not None, it will be called with parts of the raw + message as it is unpacked. + + When there are not enough bytes for unpacking, `unpack()` raises + `OutOfData` Exception. """ return self._unpack(template_construct, write_bytes) @@ -531,16 +536,30 @@ cdef class Unpacker(object): """ read and ignore one object, returning None - If write_bytes is not None, it will be called with parts of the raw message as it is unpacked. + If write_bytes is not None, it will be called with parts of the raw + message as it is unpacked. + + When there are not enough bytes for unpacking, `unpack()` raises + `OutOfData` Exception. """ return self._unpack(template_skip, write_bytes) def read_array_header(self, object write_bytes=None): - """assuming the next object is an array, return its size n, such that the next n unpack() calls will iterate over its contents.""" + """assuming the next object is an array, return its size n, such that + the next n unpack() calls will iterate over its contents. + + When there are not enough bytes for unpacking, `unpack()` raises + `OutOfData` Exception. + """ return self._unpack(read_array_header, write_bytes) def read_map_header(self, object write_bytes=None): - """assuming the next object is a map, return its size n, such that the next n * 2 unpack() calls will iterate over its key-value pairs.""" + """assuming the next object is a map, return its size n, such that the + next n * 2 unpack() calls will iterate over its key-value pairs. + + When there are not enough bytes for unpacking, `unpack()` raises + `OutOfData` Exception. + """ return self._unpack(read_map_header, write_bytes) def __iter__(self): From ed40c671dac175e437fe67689053b5fe8a269ad3 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 10 Dec 2012 01:42:38 +0900 Subject: [PATCH 0617/1172] `pack` raise MemoryError when realloc is failed. --- msgpack/_msgpack.pyx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 2c81cb4..b2b5222 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -179,7 +179,9 @@ cdef class Packer(object): cpdef pack(self, object obj): cdef int ret ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) - if ret: + if ret == -1: + raise MemoryError + elif ret: # should not happen. raise TypeError buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 From 30025c7ea0a53e17a907ad35ffa56b7b7efd99d1 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 10 Dec 2012 20:06:00 +0900 Subject: [PATCH 0618/1172] Improve docstring. --- msgpack/_msgpack.pyx | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index b2b5222..74a4cc4 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -529,8 +529,7 @@ cdef class Unpacker(object): If write_bytes is not None, it will be called with parts of the raw message as it is unpacked. - When there are not enough bytes for unpacking, `unpack()` raises - `OutOfData` Exception. + Raises `OutOfData` when there are no more bytes to unpack. """ return self._unpack(template_construct, write_bytes) @@ -541,8 +540,7 @@ cdef class Unpacker(object): If write_bytes is not None, it will be called with parts of the raw message as it is unpacked. - When there are not enough bytes for unpacking, `unpack()` raises - `OutOfData` Exception. + Raises `OutOfData` when there are no more bytes to unpack. """ return self._unpack(template_skip, write_bytes) @@ -550,8 +548,7 @@ cdef class Unpacker(object): """assuming the next object is an array, return its size n, such that the next n unpack() calls will iterate over its contents. - When there are not enough bytes for unpacking, `unpack()` raises - `OutOfData` Exception. + Raises `OutOfData` when there are no more bytes to unpack. """ return self._unpack(read_array_header, write_bytes) @@ -559,8 +556,7 @@ cdef class Unpacker(object): """assuming the next object is a map, return its size n, such that the next n * 2 unpack() calls will iterate over its key-value pairs. - When there are not enough bytes for unpacking, `unpack()` raises - `OutOfData` Exception. + Raises `OutOfData` when there are no more bytes to unpack. """ return self._unpack(read_map_header, write_bytes) From 1c0fe10a2fd5a32cf3f558683695220f3718a4b8 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 10 Dec 2012 20:12:38 +0900 Subject: [PATCH 0619/1172] Remove unused UnpackException. --- msgpack/_msgpack.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 74a4cc4..52d63cc 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -39,7 +39,6 @@ cdef int DEFAULT_RECURSE_LIMIT=511 from msgpack.exceptions import ( - UnpackException, BufferFull, OutOfData, UnpackValueError, From 0c7ab7c3441e96b33fa7bca8fcaa10d3087aba2a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 10 Dec 2012 20:17:18 +0900 Subject: [PATCH 0620/1172] refactoring: remove pack_define.h --- msgpack/pack.h | 3 ++- msgpack/pack_define.h | 26 -------------------------- 2 files changed, 2 insertions(+), 27 deletions(-) delete mode 100644 msgpack/pack_define.h diff --git a/msgpack/pack.h b/msgpack/pack.h index 4c0373e..bb939d9 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -19,7 +19,8 @@ #include #include #include "sysdep.h" -#include "pack_define.h" +#include +#include #ifdef __cplusplus extern "C" { diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h deleted file mode 100644 index 4845d52..0000000 --- a/msgpack/pack_define.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * MessagePack unpacking routine template - * - * Copyright (C) 2008-2010 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MSGPACK_PACK_DEFINE_H__ -#define MSGPACK_PACK_DEFINE_H__ - -#include "msgpack/sysdep.h" -#include -#include - -#endif /* msgpack/pack_define.h */ - From 537a2ab3f262d65f5a85741ddbb4ac6604ed7e0b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 10 Dec 2012 21:26:41 +0900 Subject: [PATCH 0621/1172] Add `Packer.pack_pairs`. --- msgpack/_msgpack.pyx | 39 +++++++++++++++++++++++++++++++++++---- test/test_pack.py | 11 +++++++++-- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 52d63cc..2d97f21 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -186,18 +186,49 @@ cdef class Packer(object): self.pk.length = 0 return buf - cpdef pack_array_header(self, size_t size): - msgpack_pack_array(&self.pk, size) + def pack_array_header(self, size_t size): + cdef int ret = msgpack_pack_array(&self.pk, size) + if ret == -1: + raise MemoryError + elif ret: # should not happen + raise TypeError buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 return buf - cpdef pack_map_header(self, size_t size): - msgpack_pack_map(&self.pk, size) + def pack_map_header(self, size_t size): + cdef int ret = msgpack_pack_map(&self.pk, size) + if ret == -1: + raise MemoryError + elif ret: # should not happen + raise TypeError buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 return buf + def pack_map_pairs(self, object pairs): + """ + Pack *pairs* as msgpack map type. + + *pairs* should sequence of pair. + (`len(pairs)` and `for k, v in *pairs*:` should be supported.) + """ + cdef int ret = msgpack_pack_map(&self.pk, len(pairs)) + if ret == 0: + for k, v in pairs: + ret = self._pack(k) + if ret != 0: break + ret = self._pack(v) + if ret != 0: break + if ret == -1: + raise MemoryError + elif ret: # should not happen + raise TypeError + buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) + self.pk.length = 0 + return buf + + def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'): """ pack an object `o` and write it to stream).""" diff --git a/test/test_pack.py b/test/test_pack.py index 21c2bd7..b918f8e 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -118,8 +118,6 @@ def testMapSize(sizes=[0, 5, 50, 1000]): assert unpacker.unpack() == dict((i, i * 2) for i in range(size)) - - class odict(dict): '''Reimplement OrderedDict to run test on Python 2.6''' def __init__(self, seq): @@ -144,5 +142,14 @@ def test_odict(): assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1), seq) +def test_pairlist(): + pairlist = [(b'a', 1), (2, b'b'), (b'foo', b'bar')] + packer = Packer() + packed = packer.pack_map_pairs(pairlist) + unpacked = unpackb(packed, object_pairs_hook=list) + assert pairlist == unpacked + + + if __name__ == '__main__': main() From 4adc6f194d9986eee4cfa4999d30862e4251d4cf Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 10 Dec 2012 21:47:18 +0900 Subject: [PATCH 0622/1172] Add `autoreset` option to Packer. --- msgpack/_msgpack.pyx | 48 +++++++++++++++++++++++++++++++++----------- test/test_pack.py | 15 ++++++++++++++ 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 2d97f21..d58255e 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -54,6 +54,16 @@ cdef class Packer(object): packer = Packer() astream.write(packer.pack(a)) astream.write(packer.pack(b)) + + Packer's constructor has some keyword arguments: + + * *defaut* - Convert user type to builtin type that Packer supports. + See also simplejson's document. + * *encoding* - Convert unicode to bytes with this encoding. (default: 'utf-8') + * *unicode_erros* - Error handler for encoding unicode. (default: 'strict') + * *use_single_float* - Use single precision float type for float. (default: False) + * *autoreset* - Reset buffer after each pack and return it's content as `bytes`. (default: True). + If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. """ cdef msgpack_packer pk cdef object _default @@ -62,6 +72,7 @@ cdef class Packer(object): cdef char *encoding cdef char *unicode_errors cdef bool use_float + cdef bint autoreset def __cinit__(self): cdef int buf_size = 1024*1024 @@ -71,8 +82,9 @@ cdef class Packer(object): self.pk.buf_size = buf_size self.pk.length = 0 - def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False): + def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, bint autoreset=1): self.use_float = use_single_float + self.autoreset = autoreset if default is not None: if not PyCallable_Check(default): raise TypeError("default must be a callable.") @@ -182,9 +194,10 @@ cdef class Packer(object): raise MemoryError elif ret: # should not happen. raise TypeError - buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - self.pk.length = 0 - return buf + if self.autoreset: + buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) + self.pk.length = 0 + return buf def pack_array_header(self, size_t size): cdef int ret = msgpack_pack_array(&self.pk, size) @@ -192,9 +205,10 @@ cdef class Packer(object): raise MemoryError elif ret: # should not happen raise TypeError - buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - self.pk.length = 0 - return buf + if self.autoreset: + buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) + self.pk.length = 0 + return buf def pack_map_header(self, size_t size): cdef int ret = msgpack_pack_map(&self.pk, size) @@ -202,9 +216,10 @@ cdef class Packer(object): raise MemoryError elif ret: # should not happen raise TypeError - buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - self.pk.length = 0 - return buf + if self.autoreset: + buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) + self.pk.length = 0 + return buf def pack_map_pairs(self, object pairs): """ @@ -224,9 +239,18 @@ cdef class Packer(object): raise MemoryError elif ret: # should not happen raise TypeError - buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) + if self.autoreset: + buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) + self.pk.length = 0 + return buf + + def reset(self): + """Clear internal buffer.""" self.pk.length = 0 - return buf + + def bytes(self): + """Return buffer content.""" + return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'): diff --git a/test/test_pack.py b/test/test_pack.py index b918f8e..b934dd2 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -103,6 +103,21 @@ def testArraySize(sizes=[0, 5, 50, 1000]): for size in sizes: assert unpacker.unpack() == list(range(size)) +def test_manualreset(sizes=[0, 5, 50, 1000]): + packer = Packer(autoreset=False) + for size in sizes: + packer.pack_array_header(size) + for i in range(size): + packer.pack(i) + + bio = six.BytesIO(packer.bytes()) + unpacker = Unpacker(bio, use_list=1) + for size in sizes: + assert unpacker.unpack() == list(range(size)) + + packer.reset() + assert packer.bytes() == b'' + def testMapSize(sizes=[0, 5, 50, 1000]): bio = six.BytesIO() packer = Packer() From 171145e56290a2254d6b648d438dde33cd631fc4 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 10 Dec 2012 21:54:17 +0900 Subject: [PATCH 0623/1172] Update changelog. --- ChangeLog.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 279cb81..a66a276 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -13,6 +13,12 @@ Changes * Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman) * Add capturing feature. You can pass the writable object to ``Unpacker.unpack()`` as a second parameter. +* Add ``Packer.pack_array_header`` and ``Packer.pack_map_header``. + These methods only pack header of each type. +* Add ``autoreset`` option to ``Packer`` (default: True). + Packer doesn't return packed bytes and clear internal buffer. +* Add ``Packer.pack_map_pairs``. It packs sequence of pair to map type. + 0.2.3 ======= From 3478406537fc36badb2501d2110a2bceb48a45b6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 02:46:13 +0900 Subject: [PATCH 0624/1172] Fix tests. --- msgpack/_msgpack.pyx | 62 ++++++++++++++++++++++++------------------ setup.py | 2 +- test/test_obj.py | 2 +- test/test_sequnpack.py | 2 +- 4 files changed, 39 insertions(+), 29 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index dca6237..5feba11 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -1,21 +1,17 @@ # coding: utf-8 #cython: embedsignature=True -import warnings - from cpython cimport * cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef char* const_void_ptr "const void*" ctypedef struct PyObject cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1 - char* __FILE__ - int __LINE__ from libc.stdlib cimport * from libc.string cimport * from libc.limits cimport * - +import warnings cdef extern from "pack.h": struct msgpack_packer: @@ -218,7 +214,9 @@ cdef extern from "unpack.h": void template_init(template_context* ctx) object template_data(template_context* ctx) -cdef inline init_ctx(template_context *ctx, object object_hook, object object_pairs_hook, object list_hook, bint use_list, encoding, unicode_errors): +cdef inline init_ctx(template_context *ctx, + object object_hook, object object_pairs_hook, object list_hook, + bint use_list, char* encoding, char* unicode_errors): template_init(ctx) ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL @@ -244,20 +242,8 @@ cdef inline init_ctx(template_context *ctx, object object_hook, object object_pa raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook - if encoding is None: - ctx.user.encoding = NULL - ctx.user.unicode_errors = NULL - else: - if isinstance(encoding, unicode): - _bencoding = encoding.encode('ascii') - else: - _bencoding = encoding - ctx.user.encoding = PyBytes_AsString(_bencoding) - if isinstance(unicode_errors, unicode): - _berrors = unicode_errors.encode('ascii') - else: - _berrors = unicode_errors - ctx.user.unicode_errors = PyBytes_AsString(_berrors) + ctx.user.encoding = encoding + ctx.user.unicode_errors = unicode_errors def unpackb(object packed, object object_hook=None, object list_hook=None, use_list=None, encoding=None, unicode_errors="strict", @@ -273,13 +259,26 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef char* buf cdef Py_ssize_t buf_len + cdef char* cenc = NULL + cdef char* cerr = NULL PyObject_AsReadBuffer(packed, &buf, &buf_len) if use_list is None: warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) use_list = 0 - init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors) + + if encoding is not None: + if isinstance(encoding, unicode): + encoding = encoding.encode('ascii') + cenc = PyBytes_AsString(encoding) + + if unicode_errors is not None: + if isinstance(unicode_errors, unicode): + unicode_errors = unicode_errors.encode('ascii') + cerr = PyBytes_AsString(unicode_errors) + + init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) ret = template_execute(&ctx, buf, buf_len, &off, 1) if ret == 1: obj = template_data(&ctx) @@ -361,10 +360,7 @@ cdef class Unpacker(object): cdef object file_like_read cdef Py_ssize_t read_size cdef object object_hook - cdef object _bencoding - cdef object _berrors - cdef char *encoding - cdef char *unicode_errors + cdef object encoding, unicode_errors cdef size_t max_buffer_size def __cinit__(self): @@ -378,6 +374,7 @@ cdef class Unpacker(object): object object_hook=None, object object_pairs_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, ): + cdef char *cenc=NULL, *cerr=NULL if use_list is None: warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1) use_list = 0 @@ -401,7 +398,20 @@ cdef class Unpacker(object): self.buf_size = read_size self.buf_head = 0 self.buf_tail = 0 - init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors) + + if encoding is not None: + if isinstance(encoding, unicode): + encoding = encoding.encode('ascii') + self.encoding = encoding + cenc = PyBytes_AsString(encoding) + + if unicode_errors is not None: + if isinstance(unicode_errors, unicode): + unicode_errors = unicode_errors.encode('ascii') + self.unicode_errors = unicode_errors + cerr = PyBytes_AsString(unicode_errors) + + init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) def feed(self, object next_bytes): cdef char* buf diff --git a/setup.py b/setup.py index 9f0ce5d..86b0b34 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ except ImportError: def cythonize(src): sys.stderr.write("cythonize: %r\n" % (src,)) - cython_compiler.compile([src], emit_linenums=True) + cython_compiler.compile([src]) def ensure_source(src): pyx = os.path.splitext(src)[0] + '.pyx' diff --git a/test/test_obj.py b/test/test_obj.py index 881e627..c38d6dc 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -34,7 +34,7 @@ def test_decode_pairs_hook(): @raises(ValueError) def test_only_one_obj_hook(): - unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x) + unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x, use_list=1) @raises(ValueError) def test_bad_hook(): diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index dac36a8..eac0828 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -44,7 +44,7 @@ def test_foobar_skip(): assert 1, "ok" def test_maxbuffersize(): - nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3) + nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3, use_list=1) unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1) unpacker.feed(b'fo') nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob') From 6ae363ea27e409c63e951eb32d7c16df0309bb52 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 02:52:24 +0900 Subject: [PATCH 0625/1172] Update changelog for 0.2.3 --- ChangeLog.rst | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 6a4d27b..833c97c 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,12 +1,3 @@ -0.3.0 -===== -:release date: in development - -Changes -------- -* Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman) - - 0.2.3 ======= :release date: in development @@ -14,6 +5,7 @@ Changes Changes ------- * Warn when use_list is not specified. It's default value will be changed in 0.3. +* Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman) Bugs fixed ----------- From 4a20700e2000d4baa952e8db48801672e53bab99 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 02:56:20 +0900 Subject: [PATCH 0626/1172] prepare 0.2.3 --- README.rst | 4 ++-- msgpack/_version.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 1da6ece..5c0829a 100644 --- a/README.rst +++ b/README.rst @@ -3,8 +3,8 @@ MessagePack Python Binding =========================== :author: INADA Naoki -:version: 0.2.0 -:date: 2012-06-27 +:version: 0.2.3 +:date: 2012-12-11 HOW TO USE ----------- diff --git a/msgpack/_version.py b/msgpack/_version.py index 9bffd02..1bbdf2a 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 2, 3, 'dev1') +version = (0, 2, 3) From cb7dff331911f836f9bfac2e5f0450760e362e89 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 02:59:45 +0900 Subject: [PATCH 0627/1172] Add .travis.yml --- .travis.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..11835cb --- /dev/null +++ b/.travis.yml @@ -0,0 +1,14 @@ +language: python +python: + - 2.6 + - 2.7 + - 3.2 + +install: + - sudo apt-get update -qq + - sudo apt-get install -q cython + - cython --cplus msgpack/_msgpack.pyx + - pip install six --use-mirrors + - python setup.py install + +script: "nosetests -w test" From e9f9e9e83ff5b69e7aab5384d9cb507363dd88e3 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 09:26:15 +0900 Subject: [PATCH 0628/1172] Don't use c++ on travis. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 11835cb..e8c2cec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ python: install: - sudo apt-get update -qq - sudo apt-get install -q cython - - cython --cplus msgpack/_msgpack.pyx + - cython msgpack/_msgpack.pyx - pip install six --use-mirrors - python setup.py install From 280d56eb9b61f68ff87e3e217eeec0b5fd645c38 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 22:05:00 +0900 Subject: [PATCH 0629/1172] rename _msgpack.pyx => _packer.pyx --- msgpack/__init__.py | 3 ++- msgpack/{_msgpack.pyx => _packer.pyx} | 0 setup.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) rename msgpack/{_msgpack.pyx => _packer.pyx} (100%) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 98b1ab7..1aacee1 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,6 +1,7 @@ # coding: utf-8 from msgpack._version import version -from msgpack._msgpack import * +from msgpack.exceptions import * +from msgpack._packer import pack, packb, Packer, unpack, unpackb, Unpacker # alias for compatibility to simplejson/marshal/pickle. load = unpack diff --git a/msgpack/_msgpack.pyx b/msgpack/_packer.pyx similarity index 100% rename from msgpack/_msgpack.pyx rename to msgpack/_packer.pyx diff --git a/setup.py b/setup.py index b9af8c3..5a342e0 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ if have_cython: else: Sdist = sdist -sources = ['msgpack/_msgpack.cpp'] +sources = ['msgpack/_packer.cpp'] libraries = [] if sys.platform == 'win32': libraries.append('ws2_32') @@ -69,7 +69,7 @@ if sys.byteorder == 'big': else: macros = [('__LITTLE_ENDIAN__', '1')] -msgpack_mod = Extension('msgpack._msgpack', +msgpack_mod = Extension('msgpack._packer', sources=sources, libraries=libraries, include_dirs=['.'], From eec02b87294a84199d3eb527b785d6091307d868 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 22:05:48 +0900 Subject: [PATCH 0630/1172] remove unused import --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 5a342e0..c73b16e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,6 @@ # coding: utf-8 import os import sys -import shutil from glob import glob from distutils.command.sdist import sdist from setuptools import setup, Extension From b79e5ba4e5e16eb9907e2e0081593ce6aa14eb66 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 22:15:21 +0900 Subject: [PATCH 0631/1172] Split _msgpack.pyx --- msgpack/__init__.py | 3 +- msgpack/_packer.pyx | 374 +---------------------------------------- msgpack/_unpacker.pyx | 382 ++++++++++++++++++++++++++++++++++++++++++ msgpack/exceptions.py | 6 + setup.py | 24 ++- 5 files changed, 407 insertions(+), 382 deletions(-) create mode 100644 msgpack/_unpacker.pyx diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 1aacee1..0bfe6fe 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,7 +1,8 @@ # coding: utf-8 from msgpack._version import version from msgpack.exceptions import * -from msgpack._packer import pack, packb, Packer, unpack, unpackb, Unpacker +from msgpack._packer import pack, packb, Packer +from msgpack._unpacker import unpack, unpackb, Unpacker # alias for compatibility to simplejson/marshal/pickle. load = unpack diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index d58255e..a5bc570 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -7,13 +7,12 @@ cdef extern from "Python.h": ctypedef char* const_void_ptr "const void*" ctypedef struct PyObject cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1 - char* __FILE__ - int __LINE__ from libc.stdlib cimport * from libc.string cimport * from libc.limits cimport * +from msgpack.exceptions import PackValueError cdef extern from "pack.h": struct msgpack_packer: @@ -38,13 +37,6 @@ cdef extern from "pack.h": cdef int DEFAULT_RECURSE_LIMIT=511 -from msgpack.exceptions import ( - BufferFull, - OutOfData, - UnpackValueError, - ExtraData, - ) - cdef class Packer(object): """MessagePack Packer @@ -118,7 +110,7 @@ cdef class Packer(object): cdef dict d if nest_limit < 0: - raise UnpackValueError("recursion limit exceeded.") + raise PackValueError("recursion limit exceeded.") if o is None: ret = msgpack_pack_nil(&self.pk) @@ -265,365 +257,3 @@ def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors, use_single_float=use_single_float) return packer.pack(o) - - -cdef extern from "unpack.h": - ctypedef struct msgpack_user: - bint use_list - PyObject* object_hook - bint has_pairs_hook # call object_hook with k-v pairs - PyObject* list_hook - char *encoding - char *unicode_errors - - ctypedef struct template_context: - msgpack_user user - PyObject* obj - size_t count - unsigned int ct - PyObject* key - - ctypedef int (*execute_fn)(template_context* ctx, const_char_ptr data, - size_t len, size_t* off) except -1 - execute_fn template_construct - execute_fn template_skip - execute_fn read_array_header - execute_fn read_map_header - void template_init(template_context* ctx) - object template_data(template_context* ctx) - -cdef inline init_ctx(template_context *ctx, - object object_hook, object object_pairs_hook, object list_hook, - bint use_list, char* encoding, char* unicode_errors): - template_init(ctx) - ctx.user.use_list = use_list - ctx.user.object_hook = ctx.user.list_hook = NULL - - if object_hook is not None and object_pairs_hook is not None: - raise ValueError("object_pairs_hook and object_hook are mutually exclusive.") - - if object_hook is not None: - if not PyCallable_Check(object_hook): - raise TypeError("object_hook must be a callable.") - ctx.user.object_hook = object_hook - - if object_pairs_hook is None: - ctx.user.has_pairs_hook = False - else: - if not PyCallable_Check(object_pairs_hook): - raise TypeError("object_pairs_hook must be a callable.") - ctx.user.object_hook = object_pairs_hook - ctx.user.has_pairs_hook = True - - if list_hook is not None: - if not PyCallable_Check(list_hook): - raise TypeError("list_hook must be a callable.") - ctx.user.list_hook = list_hook - - ctx.user.encoding = encoding - ctx.user.unicode_errors = unicode_errors - -def unpackb(object packed, object object_hook=None, object list_hook=None, - bint use_list=1, encoding=None, unicode_errors="strict", - object_pairs_hook=None, - ): - """Unpack packed_bytes to object. Returns an unpacked object. - - Raises `ValueError` when `packed` contains extra bytes. - """ - cdef template_context ctx - cdef size_t off = 0 - cdef int ret - - cdef char* buf - cdef Py_ssize_t buf_len - cdef char* cenc = NULL - cdef char* cerr = NULL - - PyObject_AsReadBuffer(packed, &buf, &buf_len) - - if encoding is not None: - if isinstance(encoding, unicode): - encoding = encoding.encode('ascii') - cenc = PyBytes_AsString(encoding) - - if unicode_errors is not None: - if isinstance(unicode_errors, unicode): - unicode_errors = unicode_errors.encode('ascii') - cerr = PyBytes_AsString(unicode_errors) - - init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) - ret = template_construct(&ctx, buf, buf_len, &off) - if ret == 1: - obj = template_data(&ctx) - if off < buf_len: - raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) - return obj - else: - return None - - -def unpack(object stream, object object_hook=None, object list_hook=None, - bint use_list=1, encoding=None, unicode_errors="strict", - object_pairs_hook=None, - ): - """Unpack an object from `stream`. - - Raises `ValueError` when `stream` has extra bytes. - """ - return unpackb(stream.read(), use_list=use_list, - object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, - encoding=encoding, unicode_errors=unicode_errors, - ) - - -cdef class Unpacker(object): - """ - Streaming unpacker. - - `file_like` is a file-like object having `.read(n)` method. - When `Unpacker` initialized with `file_like`, unpacker reads serialized data - from it and `.feed()` method is not usable. - - `read_size` is used as `file_like.read(read_size)`. - (default: min(1024**2, max_buffer_size)) - - If `use_list` is true (default), msgpack list is deserialized to Python list. - Otherwise, it is deserialized to Python tuple. - - `object_hook` is same to simplejson. If it is not None, it should be callable - and Unpacker calls it with a dict argument after deserializing a map. - - `object_pairs_hook` is same to simplejson. If it is not None, it should be callable - and Unpacker calls it with a list of key-value pairs after deserializing a map. - - `encoding` is encoding used for decoding msgpack bytes. If it is None (default), - msgpack bytes is deserialized to Python bytes. - - `unicode_errors` is used for decoding bytes. - - `max_buffer_size` limits size of data waiting unpacked. - 0 means system's INT_MAX (default). - Raises `BufferFull` exception when it is insufficient. - You shoud set this parameter when unpacking data from untrasted source. - - example of streaming deserialize from file-like object:: - - unpacker = Unpacker(file_like) - for o in unpacker: - do_something(o) - - example of streaming deserialize from socket:: - - unpacker = Unpacker() - while 1: - buf = sock.recv(1024**2) - if not buf: - break - unpacker.feed(buf) - for o in unpacker: - do_something(o) - """ - cdef template_context ctx - cdef char* buf - cdef size_t buf_size, buf_head, buf_tail - cdef object file_like - cdef object file_like_read - cdef Py_ssize_t read_size - cdef object object_hook - cdef object encoding, unicode_errors - cdef size_t max_buffer_size - - def __cinit__(self): - self.buf = NULL - - def __dealloc__(self): - free(self.buf) - self.buf = NULL - - def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1, - object object_hook=None, object object_pairs_hook=None, object list_hook=None, - encoding=None, unicode_errors='strict', int max_buffer_size=0, - ): - cdef char *cenc=NULL, *cerr=NULL - - self.file_like = file_like - if file_like: - self.file_like_read = file_like.read - if not PyCallable_Check(self.file_like_read): - raise ValueError("`file_like.read` must be a callable.") - if not max_buffer_size: - max_buffer_size = INT_MAX - if read_size > max_buffer_size: - raise ValueError("read_size should be less or equal to max_buffer_size") - if not read_size: - read_size = min(max_buffer_size, 1024**2) - self.max_buffer_size = max_buffer_size - self.read_size = read_size - self.buf = malloc(read_size) - if self.buf == NULL: - raise MemoryError("Unable to allocate internal buffer.") - self.buf_size = read_size - self.buf_head = 0 - self.buf_tail = 0 - - if encoding is not None: - if isinstance(encoding, unicode): - encoding = encoding.encode('ascii') - self.encoding = encoding - cenc = PyBytes_AsString(encoding) - - if unicode_errors is not None: - if isinstance(unicode_errors, unicode): - unicode_errors = unicode_errors.encode('ascii') - self.unicode_errors = unicode_errors - cerr = PyBytes_AsString(unicode_errors) - - init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) - - def feed(self, object next_bytes): - """Append `next_bytes` to internal buffer.""" - cdef char* buf - cdef Py_ssize_t buf_len - if self.file_like is not None: - raise AssertionError( - "unpacker.feed() is not be able to use with `file_like`.") - PyObject_AsReadBuffer(next_bytes, &buf, &buf_len) - self.append_buffer(buf, buf_len) - - cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len): - cdef: - char* buf = self.buf - char* new_buf - size_t head = self.buf_head - size_t tail = self.buf_tail - size_t buf_size = self.buf_size - size_t new_size - - if tail + _buf_len > buf_size: - if ((tail - head) + _buf_len) <= buf_size: - # move to front. - memmove(buf, buf + head, tail - head) - tail -= head - head = 0 - else: - # expand buffer. - new_size = (tail-head) + _buf_len - if new_size > self.max_buffer_size: - raise BufferFull - new_size = min(new_size*2, self.max_buffer_size) - new_buf = malloc(new_size) - if new_buf == NULL: - # self.buf still holds old buffer and will be freed during - # obj destruction - raise MemoryError("Unable to enlarge internal buffer.") - memcpy(new_buf, buf + head, tail - head) - free(buf) - - buf = new_buf - buf_size = new_size - tail -= head - head = 0 - - memcpy(buf + tail, (_buf), _buf_len) - self.buf = buf - self.buf_head = head - self.buf_size = buf_size - self.buf_tail = tail + _buf_len - - cdef read_from_file(self): - next_bytes = self.file_like_read( - min(self.read_size, - self.max_buffer_size - (self.buf_tail - self.buf_head) - )) - if next_bytes: - self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes)) - else: - self.file_like = None - - cdef object _unpack(self, execute_fn execute, object write_bytes, bint iter=0): - cdef int ret - cdef object obj - cdef size_t prev_head - while 1: - prev_head = self.buf_head - ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) - if write_bytes is not None: - write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head)) - - if ret == 1: - obj = template_data(&self.ctx) - template_init(&self.ctx) - return obj - elif ret == 0: - if self.file_like is not None: - self.read_from_file() - continue - if iter: - raise StopIteration("No more data to unpack.") - else: - raise OutOfData("No more data to unpack.") - else: - raise ValueError("Unpack failed: error = %d" % (ret,)) - - def read_bytes(self, Py_ssize_t nbytes): - """read a specified number of raw bytes from the stream""" - cdef size_t nread - nread = min(self.buf_tail - self.buf_head, nbytes) - ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread) - self.buf_head += nread - if len(ret) < nbytes and self.file_like is not None: - ret += self.file_like.read(nbytes - len(ret)) - return ret - - def unpack(self, object write_bytes=None): - """ - unpack one object - - If write_bytes is not None, it will be called with parts of the raw - message as it is unpacked. - - Raises `OutOfData` when there are no more bytes to unpack. - """ - return self._unpack(template_construct, write_bytes) - - def skip(self, object write_bytes=None): - """ - read and ignore one object, returning None - - If write_bytes is not None, it will be called with parts of the raw - message as it is unpacked. - - Raises `OutOfData` when there are no more bytes to unpack. - """ - return self._unpack(template_skip, write_bytes) - - def read_array_header(self, object write_bytes=None): - """assuming the next object is an array, return its size n, such that - the next n unpack() calls will iterate over its contents. - - Raises `OutOfData` when there are no more bytes to unpack. - """ - return self._unpack(read_array_header, write_bytes) - - def read_map_header(self, object write_bytes=None): - """assuming the next object is a map, return its size n, such that the - next n * 2 unpack() calls will iterate over its key-value pairs. - - Raises `OutOfData` when there are no more bytes to unpack. - """ - return self._unpack(read_map_header, write_bytes) - - def __iter__(self): - return self - - def __next__(self): - return self._unpack(template_construct, None, 1) - - # for debug. - #def _buf(self): - # return PyString_FromStringAndSize(self.buf, self.buf_tail) - - #def _off(self): - # return self.buf_head - diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx new file mode 100644 index 0000000..4a27d9e --- /dev/null +++ b/msgpack/_unpacker.pyx @@ -0,0 +1,382 @@ +# coding: utf-8 +#cython: embedsignature=True + +from cpython cimport * +cdef extern from "Python.h": + ctypedef char* const_char_ptr "const char*" + ctypedef char* const_void_ptr "const void*" + ctypedef struct PyObject + cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1 + +from libc.stdlib cimport * +from libc.string cimport * +from libc.limits cimport * + +from msgpack.exceptions import ( + BufferFull, + OutOfData, + UnpackValueError, + ExtraData, + ) + + + +cdef extern from "unpack.h": + ctypedef struct msgpack_user: + bint use_list + PyObject* object_hook + bint has_pairs_hook # call object_hook with k-v pairs + PyObject* list_hook + char *encoding + char *unicode_errors + + ctypedef struct template_context: + msgpack_user user + PyObject* obj + size_t count + unsigned int ct + PyObject* key + + ctypedef int (*execute_fn)(template_context* ctx, const_char_ptr data, + size_t len, size_t* off) except -1 + execute_fn template_construct + execute_fn template_skip + execute_fn read_array_header + execute_fn read_map_header + void template_init(template_context* ctx) + object template_data(template_context* ctx) + +cdef inline init_ctx(template_context *ctx, + object object_hook, object object_pairs_hook, object list_hook, + bint use_list, char* encoding, char* unicode_errors): + template_init(ctx) + ctx.user.use_list = use_list + ctx.user.object_hook = ctx.user.list_hook = NULL + + if object_hook is not None and object_pairs_hook is not None: + raise ValueError("object_pairs_hook and object_hook are mutually exclusive.") + + if object_hook is not None: + if not PyCallable_Check(object_hook): + raise TypeError("object_hook must be a callable.") + ctx.user.object_hook = object_hook + + if object_pairs_hook is None: + ctx.user.has_pairs_hook = False + else: + if not PyCallable_Check(object_pairs_hook): + raise TypeError("object_pairs_hook must be a callable.") + ctx.user.object_hook = object_pairs_hook + ctx.user.has_pairs_hook = True + + if list_hook is not None: + if not PyCallable_Check(list_hook): + raise TypeError("list_hook must be a callable.") + ctx.user.list_hook = list_hook + + ctx.user.encoding = encoding + ctx.user.unicode_errors = unicode_errors + +def unpackb(object packed, object object_hook=None, object list_hook=None, + bint use_list=1, encoding=None, unicode_errors="strict", + object_pairs_hook=None, + ): + """Unpack packed_bytes to object. Returns an unpacked object. + + Raises `ValueError` when `packed` contains extra bytes. + """ + cdef template_context ctx + cdef size_t off = 0 + cdef int ret + + cdef char* buf + cdef Py_ssize_t buf_len + cdef char* cenc = NULL + cdef char* cerr = NULL + + PyObject_AsReadBuffer(packed, &buf, &buf_len) + + if encoding is not None: + if isinstance(encoding, unicode): + encoding = encoding.encode('ascii') + cenc = PyBytes_AsString(encoding) + + if unicode_errors is not None: + if isinstance(unicode_errors, unicode): + unicode_errors = unicode_errors.encode('ascii') + cerr = PyBytes_AsString(unicode_errors) + + init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) + ret = template_construct(&ctx, buf, buf_len, &off) + if ret == 1: + obj = template_data(&ctx) + if off < buf_len: + raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) + return obj + else: + raise UnpackValueError + + +def unpack(object stream, object object_hook=None, object list_hook=None, + bint use_list=1, encoding=None, unicode_errors="strict", + object_pairs_hook=None, + ): + """Unpack an object from `stream`. + + Raises `ValueError` when `stream` has extra bytes. + """ + return unpackb(stream.read(), use_list=use_list, + object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, + encoding=encoding, unicode_errors=unicode_errors, + ) + + +cdef class Unpacker(object): + """ + Streaming unpacker. + + `file_like` is a file-like object having `.read(n)` method. + When `Unpacker` initialized with `file_like`, unpacker reads serialized data + from it and `.feed()` method is not usable. + + `read_size` is used as `file_like.read(read_size)`. + (default: min(1024**2, max_buffer_size)) + + If `use_list` is true (default), msgpack list is deserialized to Python list. + Otherwise, it is deserialized to Python tuple. + + `object_hook` is same to simplejson. If it is not None, it should be callable + and Unpacker calls it with a dict argument after deserializing a map. + + `object_pairs_hook` is same to simplejson. If it is not None, it should be callable + and Unpacker calls it with a list of key-value pairs after deserializing a map. + + `encoding` is encoding used for decoding msgpack bytes. If it is None (default), + msgpack bytes is deserialized to Python bytes. + + `unicode_errors` is used for decoding bytes. + + `max_buffer_size` limits size of data waiting unpacked. + 0 means system's INT_MAX (default). + Raises `BufferFull` exception when it is insufficient. + You shoud set this parameter when unpacking data from untrasted source. + + example of streaming deserialize from file-like object:: + + unpacker = Unpacker(file_like) + for o in unpacker: + do_something(o) + + example of streaming deserialize from socket:: + + unpacker = Unpacker() + while 1: + buf = sock.recv(1024**2) + if not buf: + break + unpacker.feed(buf) + for o in unpacker: + do_something(o) + """ + cdef template_context ctx + cdef char* buf + cdef size_t buf_size, buf_head, buf_tail + cdef object file_like + cdef object file_like_read + cdef Py_ssize_t read_size + cdef object object_hook + cdef object encoding, unicode_errors + cdef size_t max_buffer_size + + def __cinit__(self): + self.buf = NULL + + def __dealloc__(self): + free(self.buf) + self.buf = NULL + + def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1, + object object_hook=None, object object_pairs_hook=None, object list_hook=None, + encoding=None, unicode_errors='strict', int max_buffer_size=0, + ): + cdef char *cenc=NULL, *cerr=NULL + + self.file_like = file_like + if file_like: + self.file_like_read = file_like.read + if not PyCallable_Check(self.file_like_read): + raise ValueError("`file_like.read` must be a callable.") + if not max_buffer_size: + max_buffer_size = INT_MAX + if read_size > max_buffer_size: + raise ValueError("read_size should be less or equal to max_buffer_size") + if not read_size: + read_size = min(max_buffer_size, 1024**2) + self.max_buffer_size = max_buffer_size + self.read_size = read_size + self.buf = malloc(read_size) + if self.buf == NULL: + raise MemoryError("Unable to allocate internal buffer.") + self.buf_size = read_size + self.buf_head = 0 + self.buf_tail = 0 + + if encoding is not None: + if isinstance(encoding, unicode): + encoding = encoding.encode('ascii') + self.encoding = encoding + cenc = PyBytes_AsString(encoding) + + if unicode_errors is not None: + if isinstance(unicode_errors, unicode): + unicode_errors = unicode_errors.encode('ascii') + self.unicode_errors = unicode_errors + cerr = PyBytes_AsString(unicode_errors) + + init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) + + def feed(self, object next_bytes): + """Append `next_bytes` to internal buffer.""" + cdef char* buf + cdef Py_ssize_t buf_len + if self.file_like is not None: + raise AssertionError( + "unpacker.feed() is not be able to use with `file_like`.") + PyObject_AsReadBuffer(next_bytes, &buf, &buf_len) + self.append_buffer(buf, buf_len) + + cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len): + cdef: + char* buf = self.buf + char* new_buf + size_t head = self.buf_head + size_t tail = self.buf_tail + size_t buf_size = self.buf_size + size_t new_size + + if tail + _buf_len > buf_size: + if ((tail - head) + _buf_len) <= buf_size: + # move to front. + memmove(buf, buf + head, tail - head) + tail -= head + head = 0 + else: + # expand buffer. + new_size = (tail-head) + _buf_len + if new_size > self.max_buffer_size: + raise BufferFull + new_size = min(new_size*2, self.max_buffer_size) + new_buf = malloc(new_size) + if new_buf == NULL: + # self.buf still holds old buffer and will be freed during + # obj destruction + raise MemoryError("Unable to enlarge internal buffer.") + memcpy(new_buf, buf + head, tail - head) + free(buf) + + buf = new_buf + buf_size = new_size + tail -= head + head = 0 + + memcpy(buf + tail, (_buf), _buf_len) + self.buf = buf + self.buf_head = head + self.buf_size = buf_size + self.buf_tail = tail + _buf_len + + cdef read_from_file(self): + next_bytes = self.file_like_read( + min(self.read_size, + self.max_buffer_size - (self.buf_tail - self.buf_head) + )) + if next_bytes: + self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes)) + else: + self.file_like = None + + cdef object _unpack(self, execute_fn execute, object write_bytes, bint iter=0): + cdef int ret + cdef object obj + cdef size_t prev_head + while 1: + prev_head = self.buf_head + ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + if write_bytes is not None: + write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head)) + + if ret == 1: + obj = template_data(&self.ctx) + template_init(&self.ctx) + return obj + elif ret == 0: + if self.file_like is not None: + self.read_from_file() + continue + if iter: + raise StopIteration("No more data to unpack.") + else: + raise OutOfData("No more data to unpack.") + else: + raise ValueError("Unpack failed: error = %d" % (ret,)) + + def read_bytes(self, Py_ssize_t nbytes): + """read a specified number of raw bytes from the stream""" + cdef size_t nread + nread = min(self.buf_tail - self.buf_head, nbytes) + ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread) + self.buf_head += nread + if len(ret) < nbytes and self.file_like is not None: + ret += self.file_like.read(nbytes - len(ret)) + return ret + + def unpack(self, object write_bytes=None): + """ + unpack one object + + If write_bytes is not None, it will be called with parts of the raw + message as it is unpacked. + + Raises `OutOfData` when there are no more bytes to unpack. + """ + return self._unpack(template_construct, write_bytes) + + def skip(self, object write_bytes=None): + """ + read and ignore one object, returning None + + If write_bytes is not None, it will be called with parts of the raw + message as it is unpacked. + + Raises `OutOfData` when there are no more bytes to unpack. + """ + return self._unpack(template_skip, write_bytes) + + def read_array_header(self, object write_bytes=None): + """assuming the next object is an array, return its size n, such that + the next n unpack() calls will iterate over its contents. + + Raises `OutOfData` when there are no more bytes to unpack. + """ + return self._unpack(read_array_header, write_bytes) + + def read_map_header(self, object write_bytes=None): + """assuming the next object is a map, return its size n, such that the + next n * 2 unpack() calls will iterate over its key-value pairs. + + Raises `OutOfData` when there are no more bytes to unpack. + """ + return self._unpack(read_map_header, write_bytes) + + def __iter__(self): + return self + + def __next__(self): + return self._unpack(template_construct, None, 1) + + # for debug. + #def _buf(self): + # return PyString_FromStringAndSize(self.buf, self.buf_tail) + + #def _off(self): + # return self.buf_head diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py index 0a75430..2565541 100644 --- a/msgpack/exceptions.py +++ b/msgpack/exceptions.py @@ -21,3 +21,9 @@ class ExtraData(ValueError): def __str__(self): return "unpack(b) recieved extra data." + +class PackException(Exception): + pass + +class PackValueError(PackException, ValueError): + pass diff --git a/setup.py b/setup.py index c73b16e..ac3eeb5 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,6 @@ if have_cython: else: Sdist = sdist -sources = ['msgpack/_packer.cpp'] libraries = [] if sys.platform == 'win32': libraries.append('ws2_32') @@ -68,13 +67,20 @@ if sys.byteorder == 'big': else: macros = [('__LITTLE_ENDIAN__', '1')] -msgpack_mod = Extension('msgpack._packer', - sources=sources, - libraries=libraries, - include_dirs=['.'], - define_macros=macros, - ) -del sources, libraries, macros +ext_modules = [] +ext_modules.append(Extension('msgpack._packer', + sources=['msgpack/_packer.cpp'], + libraries=libraries, + include_dirs=['.'], + define_macros=macros, + )) +ext_modules.append(Extension('msgpack._unpacker', + sources=['msgpack/_unpacker.cpp'], + libraries=libraries, + include_dirs=['.'], + define_macros=macros, + )) +del libraries, macros desc = 'MessagePack (de)serializer.' @@ -88,7 +94,7 @@ setup(name='msgpack-python', author_email='songofacandy@gmail.com', version=version_str, cmdclass={'build_ext': BuildExt, 'sdist': Sdist}, - ext_modules=[msgpack_mod], + ext_modules=ext_modules, packages=['msgpack'], description=desc, long_description=long_desc, From 1c5b865db36a9cd2db5eb8de7625573a78085c56 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 22:15:53 +0900 Subject: [PATCH 0632/1172] Update .gitignore --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 7918394..1bd68b4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,5 @@ dist/* *.so *~ msgpack/__version__.py -msgpack/_msgpack.c -msgpack/_msgpack.cpp +msgpack/*.cpp *.egg-info From 78c345555b4ffd4cdba1b587e747ba59d794838d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 22:26:23 +0900 Subject: [PATCH 0633/1172] Update .travis.yml --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 11835cb..16812bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,8 @@ python: install: - sudo apt-get update -qq - sudo apt-get install -q cython - - cython --cplus msgpack/_msgpack.pyx + - cython --cplus msgpack/_packer.pyx + - cython --cplus msgpack/_unpacker.pyx - pip install six --use-mirrors - python setup.py install From 2ad02bb11a9e1a54bee11040b04a6c7138779e44 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 22:30:38 +0900 Subject: [PATCH 0634/1172] Add Python 3.3 to .travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 16812bc..dd4a10a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ python: - 2.6 - 2.7 - 3.2 + - 3.3 install: - sudo apt-get update -qq From 8b27482f5f86c91c2a0d1fc62b4d72b0accb2756 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 22:36:09 +0900 Subject: [PATCH 0635/1172] Use tox on Travis. --- .travis.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index dd4a10a..f6d8a48 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,12 @@ language: python python: - - 2.6 - 2.7 - - 3.2 - - 3.3 install: - sudo apt-get update -qq - sudo apt-get install -q cython + - sudo apt-get install -q python26-dev python27-dev python32-dev python33-dev - cython --cplus msgpack/_packer.pyx - cython --cplus msgpack/_unpacker.pyx - - pip install six --use-mirrors - - python setup.py install -script: "nosetests -w test" +script: "tox" From ef054cef51db34438aeb8df530dbf8c3ef819b64 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 22:40:07 +0900 Subject: [PATCH 0636/1172] fix package name --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f6d8a48..e636061 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ python: install: - sudo apt-get update -qq - sudo apt-get install -q cython - - sudo apt-get install -q python26-dev python27-dev python32-dev python33-dev + - sudo apt-get install -q python2.6-dev python2.7-dev python3.2-dev python3.3-dev - cython --cplus msgpack/_packer.pyx - cython --cplus msgpack/_unpacker.pyx From 1dd9280bffe2c26f0efd6e38dc212e35ed23dc0d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 22:42:08 +0900 Subject: [PATCH 0637/1172] Install tox --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e636061..ec2fa3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ python: - 2.7 install: + - pip install --use-mirrors tox - sudo apt-get update -qq - sudo apt-get install -q cython - sudo apt-get install -q python2.6-dev python2.7-dev python3.2-dev python3.3-dev From 7bebb665fbca11f9bbb83545b5d60639d26c6338 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 22:46:28 +0900 Subject: [PATCH 0638/1172] Set PIP_USE_MIRRORS=true for faster venv creation. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index ec2fa3b..efa5f8b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: python python: - 2.7 +env: + - PIP_USE_MIRRORS=true + install: - pip install --use-mirrors tox - sudo apt-get update -qq From 34611a8ccdc61ba2a8bbb5612c32ed8054315b3f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 11 Dec 2012 22:49:20 +0900 Subject: [PATCH 0639/1172] Use newer cython. --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index efa5f8b..2e6fc56 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,10 +6,9 @@ env: - PIP_USE_MIRRORS=true install: - - pip install --use-mirrors tox - sudo apt-get update -qq - - sudo apt-get install -q cython - - sudo apt-get install -q python2.6-dev python2.7-dev python3.2-dev python3.3-dev + - sudo apt-get install -q python3.3-dev + - pip install --use-mirrors tox cython - cython --cplus msgpack/_packer.pyx - cython --cplus msgpack/_unpacker.pyx From 0fa8c102d7f8ed618284e1b039fa867cfd9baa2f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 22 Dec 2012 12:12:32 +0900 Subject: [PATCH 0640/1172] Add test reproducing SEGV --- test/test_except.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/test_except.py b/test/test_except.py index ad02cb6..e142dd6 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -6,9 +6,28 @@ from msgpack import packb, unpackb import datetime +class DummyException(Exception): + pass + + def test_raise_on_find_unsupported_value(): assert_raises(TypeError, packb, datetime.datetime.now()) + +def test_raise_from_object_hook(): + def hook(obj): + raise DummyException + assert_raises(DummyException, unpackb, packb({}), object_hook=hook) + assert_raises(DummyException, unpackb, packb({'fizz': 'buzz'}), + object_hook=hook) + assert_raises(DummyException, unpackb, packb({'fizz': 'buzz'}), + object_pairs_hook=hook) + assert_raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), + object_hook=hook) + assert_raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), + object_pairs_hook=hook) + + if __name__ == '__main__': from nose import main main() From 79e44f86c9ec554f56b9807f909e21887c36d166 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 22 Dec 2012 12:14:05 +0900 Subject: [PATCH 0641/1172] Add NULL check. --- msgpack/unpack.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 7064a1b..0851ec2 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -206,6 +206,9 @@ static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_objec { if (u->object_hook) { PyObject *new_c = PyEval_CallFunction(u->object_hook, "(O)", *c); + if (new_c == NULL) { + return -1; + } Py_DECREF(*c); *c = new_c; } From ce2c5b22efcbbbc5598c7a631b3558d757b8b9dc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 22 Dec 2012 12:42:36 +0900 Subject: [PATCH 0642/1172] Check return value of _end functions. --- msgpack/unpack_template.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 6080a51..9450943 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -145,7 +145,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \ if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ if((count_) == 0) { obj = stack[top].obj; \ - construct_cb(func##_end)(user, &obj); \ + if (construct_cb(func##_end)(user, &obj) < 0) { goto _failed; } \ goto _push; } \ stack[top].ct = ct_; \ stack[top].size = count_; \ @@ -346,7 +346,7 @@ _push: if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } if(++c->count == c->size) { obj = c->obj; - construct_cb(_array_end)(user, &obj); + if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; } --top; /*printf("stack pop %d\n", top);*/ goto _push; @@ -360,7 +360,7 @@ _push: if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } if(++c->count == c->size) { obj = c->obj; - construct_cb(_map_end)(user, &obj); + if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; } --top; /*printf("stack pop %d\n", top);*/ goto _push; From d796d696d193277504b76e4ba3144154bdc1163b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 22 Dec 2012 13:09:35 +0900 Subject: [PATCH 0643/1172] revert unwanted changes. --- test/test_obj.py | 2 +- test/test_sequnpack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_obj.py b/test/test_obj.py index bfc9ab2..1d9024b 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -34,7 +34,7 @@ def test_decode_pairs_hook(): @raises(ValueError) def test_only_one_obj_hook(): - unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x, use_list=1) + unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x) @raises(ValueError) def test_bad_hook(): diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index ca4c8db..f767726 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -46,7 +46,7 @@ def test_foobar_skip(): assert 1, "ok" def test_maxbuffersize(): - nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3, use_list=1) + nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3) unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1) unpacker.feed(b'fo') nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob') From 431fe8f9e09ad8615c6dbcf8e299a8e8f2dc5ba2 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 22 Dec 2012 17:04:53 +0900 Subject: [PATCH 0644/1172] Update changelog --- ChangeLog.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 833c97c..69adcaa 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,14 @@ +0.2.4 +======= + +Bugs fixed +---------- + +* Fix SEGV when object_hook or object_pairs_hook raise Exception. (#39) + 0.2.3 ======= -:release date: in development +:release date: 2012-12-11 Changes ------- From 9dc299bd8da75e61f512e6ba50bd06face71f8a0 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 22 Dec 2012 17:11:30 +0900 Subject: [PATCH 0645/1172] 0.2.4 --- ChangeLog.rst | 1 + msgpack/_version.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 69adcaa..f684846 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,5 +1,6 @@ 0.2.4 ======= +:release date: 2012-12-22 Bugs fixed ---------- diff --git a/msgpack/_version.py b/msgpack/_version.py index 1bbdf2a..0f5f95a 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 2, 3) +version = (0, 2, 4) From 7b11a42825a631ba85608a3c0943919073d9899e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 22 Dec 2012 17:13:45 +0900 Subject: [PATCH 0646/1172] Update ChangeLog. --- ChangeLog.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index a66a276..55e296f 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -20,9 +20,19 @@ Changes * Add ``Packer.pack_map_pairs``. It packs sequence of pair to map type. + +0.2.4 +======= +:release date: 2012-12-22 + +Bugs fixed +---------- + +* Fix SEGV when object_hook or object_pairs_hook raise Exception. (#39) + 0.2.3 ======= -:release date: in development +:release date: 2012-12-11 Changes ------- From 72416e403c6d70b6fbc4ea8e05e1fe7f02a6561a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 29 Dec 2012 01:39:59 +0900 Subject: [PATCH 0647/1172] Fix unpacker doesn't raise exception for invalid input. --- msgpack/_msgpack.pyx | 4 +++- test/test_except.py | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 5feba11..78b2aa7 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -210,7 +210,7 @@ cdef extern from "unpack.h": PyObject* key int template_execute(template_context* ctx, const_char_ptr data, - size_t len, size_t* off, bint construct) except -1 + size_t len, size_t* off, bint construct) except? -1 void template_init(template_context* ctx) object template_data(template_context* ctx) @@ -285,6 +285,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, if off < buf_len: raise ValueError("Extra data.") return obj + elif ret < 0: + raise ValueError("Unpack failed: error = %d" % (ret,)) else: return None diff --git a/test/test_except.py b/test/test_except.py index e142dd6..35287df 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -6,6 +6,7 @@ from msgpack import packb, unpackb import datetime + class DummyException(Exception): pass @@ -28,6 +29,11 @@ def test_raise_from_object_hook(): object_pairs_hook=hook) +@raises(ValueError) +def test_invalidvalue(): + unpackb(b'\xd9\x97#DL_') + + if __name__ == '__main__': from nose import main main() From 593c832ab00372b4a44dd47de94e4c2546fc1193 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 29 Dec 2012 11:24:25 +0900 Subject: [PATCH 0648/1172] Use py.test instead of nosetests. --- test/test_buffer.py | 7 ++---- test/test_case.py | 15 ++++++------- test/test_except.py | 28 +++++++++--------------- test/test_format.py | 7 +----- test/test_obj.py | 39 ++++++++++++++-------------------- test/test_pack.py | 47 ++++++++++++++++++----------------------- test/test_seq.py | 14 +++--------- test/test_sequnpack.py | 25 +++++++++------------- test/test_subtype.py | 12 +++-------- test/test_unpack_raw.py | 17 ++++++--------- 10 files changed, 77 insertions(+), 134 deletions(-) diff --git a/test/test_buffer.py b/test/test_buffer.py index 785fb60..04cc02d 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -1,16 +1,13 @@ #!/usr/bin/env python # coding: utf-8 -from nose import main -from nose.tools import * from msgpack import packb, unpackb + def test_unpack_buffer(): from array import array buf = array('b') buf.fromstring(packb(('foo', 'bar'))) obj = unpackb(buf, use_list=1) - assert_equal([b'foo', b'bar'], obj) + assert [b'foo', b'bar'] == obj -if __name__ == '__main__': - main() diff --git a/test/test_case.py b/test/test_case.py index 9cbf9bd..5a4bb6c 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -1,15 +1,14 @@ #!/usr/bin/env python # coding: utf-8 -from nose import main -from nose.tools import * from msgpack import packb, unpackb def check(length, obj): v = packb(obj) - assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) - assert_equal(unpackb(v, use_list=0), obj) + assert len(v) == length, \ + "%r length should be %r but get %r" % (obj, length, len(v)) + assert unpackb(v, use_list=0) == obj def test_1(): for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1, @@ -70,8 +69,8 @@ def test_array32(): def match(obj, buf): - assert_equal(packb(obj), buf) - assert_equal(unpackb(buf, use_list=0), obj) + assert packb(obj) == buf + assert unpackb(buf, use_list=0) == obj def test_match(): cases = [ @@ -99,7 +98,5 @@ def test_match(): match(v, p) def test_unicode(): - assert_equal(b'foobar', unpackb(packb('foobar'), use_list=1)) + assert unpackb(packb('foobar'), use_list=1) == b'foobar' -if __name__ == '__main__': - main() diff --git a/test/test_except.py b/test/test_except.py index 35287df..361d4ea 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # coding: utf-8 -from nose.tools import * +from pytest import raises from msgpack import packb, unpackb import datetime @@ -12,28 +12,20 @@ class DummyException(Exception): def test_raise_on_find_unsupported_value(): - assert_raises(TypeError, packb, datetime.datetime.now()) + with raises(TypeError): + packb(datetime.datetime.now()) def test_raise_from_object_hook(): def hook(obj): raise DummyException - assert_raises(DummyException, unpackb, packb({}), object_hook=hook) - assert_raises(DummyException, unpackb, packb({'fizz': 'buzz'}), - object_hook=hook) - assert_raises(DummyException, unpackb, packb({'fizz': 'buzz'}), - object_pairs_hook=hook) - assert_raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), - object_hook=hook) - assert_raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), - object_pairs_hook=hook) + raises(DummyException, unpackb, packb({}), object_hook=hook) + raises(DummyException, unpackb, packb({'fizz': 'buzz'}), object_hook=hook) + raises(DummyException, unpackb, packb({'fizz': 'buzz'}), object_pairs_hook=hook) + raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), object_hook=hook) + raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), object_pairs_hook=hook) -@raises(ValueError) def test_invalidvalue(): - unpackb(b'\xd9\x97#DL_') - - -if __name__ == '__main__': - from nose import main - main() + with raises(ValueError): + unpackb(b'\xd9\x97#DL_') diff --git a/test/test_format.py b/test/test_format.py index ac08709..5fec0c3 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -1,12 +1,10 @@ #!/usr/bin/env python # coding: utf-8 -from nose import main -from nose.tools import * from msgpack import unpackb def check(src, should, use_list=0): - assert_equal(unpackb(src, use_list=use_list), should) + assert unpackb(src, use_list=use_list) == should def testSimpleValue(): check(b"\x93\xc0\xc2\xc3", @@ -70,6 +68,3 @@ def testMap(): b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", ({}, {None: False}, {True: False, None: False}, {}, {None: False}, {True: False, None: False})) - -if __name__ == '__main__': - main() diff --git a/test/test_obj.py b/test/test_obj.py index 1d9024b..fbf610c 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -1,9 +1,7 @@ #!/usr/bin/env python # coding: utf-8 -from nose import main -from nose.tools import * - +from pytest import raises from msgpack import packb, unpackb def _decode_complex(obj): @@ -19,27 +17,27 @@ def _encode_complex(obj): def test_encode_hook(): packed = packb([3, 1+2j], default=_encode_complex) unpacked = unpackb(packed, use_list=1) - eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2}) + assert unpacked[1] == {b'__complex__': True, b'real': 1, b'imag': 2} def test_decode_hook(): packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}]) unpacked = unpackb(packed, object_hook=_decode_complex, use_list=1) - eq_(unpacked[1], 1+2j) + assert unpacked[1] == 1+2j def test_decode_pairs_hook(): packed = packb([3, {1: 2, 3: 4}]) prod_sum = 1 * 2 + 3 * 4 unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l), use_list=1) - eq_(unpacked[1], prod_sum) + assert unpacked[1] == prod_sum -@raises(ValueError) def test_only_one_obj_hook(): - unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x) + with raises(ValueError): + unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x) -@raises(ValueError) def test_bad_hook(): - packed = packb([3, 1+2j], default=lambda o: o) - unpacked = unpackb(packed, use_list=1) + with raises(ValueError): + packed = packb([3, 1+2j], default=lambda o: o) + unpacked = unpackb(packed, use_list=1) def _arr_to_str(arr): return ''.join(str(c) for c in arr) @@ -47,7 +45,7 @@ def _arr_to_str(arr): def test_array_hook(): packed = packb([1,2,3]) unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1) - eq_(unpacked, '123') + assert unpacked == '123' class DecodeError(Exception): @@ -57,18 +55,13 @@ def bad_complex_decoder(o): raise DecodeError("Ooops!") -@raises(DecodeError) def test_an_exception_in_objecthook1(): - packed = packb({1: {'__complex__': True, 'real': 1, 'imag': 2}}) - unpackb(packed, object_hook=bad_complex_decoder) + with raises(DecodeError): + packed = packb({1: {'__complex__': True, 'real': 1, 'imag': 2}}) + unpackb(packed, object_hook=bad_complex_decoder) -@raises(DecodeError) def test_an_exception_in_objecthook2(): - packed = packb({1: [{'__complex__': True, 'real': 1, 'imag': 2}]}) - unpackb(packed, list_hook=bad_complex_decoder, use_list=1) - - - -if __name__ == '__main__': - main() + with raises(DecodeError): + packed = packb({1: [{'__complex__': True, 'real': 1, 'imag': 2}]}) + unpackb(packed, list_hook=bad_complex_decoder, use_list=1) diff --git a/test/test_pack.py b/test/test_pack.py index b934dd2..8f4117c 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -3,9 +3,7 @@ import six import struct -from nose import main -from nose.tools import * -from nose.plugins.skip import SkipTest +from pytest import raises, xfail from msgpack import packb, unpackb, Unpacker, Packer @@ -13,7 +11,7 @@ from io import BytesIO def check(data, use_list=False): re = unpackb(packb(data), use_list=use_list) - assert_equal(re, data) + assert re == data def testPack(): test_data = [ @@ -35,11 +33,11 @@ def testPackUnicode(): ] for td in test_data: re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8') - assert_equal(re, td) + assert re == td packer = Packer(encoding='utf-8') data = packer.pack(td) re = Unpacker(BytesIO(data), encoding='utf-8', use_list=1).unpack() - assert_equal(re, td) + assert re == td def testPackUTF32(): try: @@ -51,9 +49,9 @@ def testPackUTF32(): ] for td in test_data: re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32') - assert_equal(re, td) - except LookupError: - raise SkipTest + assert re == td + except LookupError as e: + xfail(e) def testPackBytes(): test_data = [ @@ -64,31 +62,31 @@ def testPackBytes(): def testIgnoreUnicodeErrors(): re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1) - assert_equal(re, "abcdef") + assert re == "abcdef" -@raises(UnicodeDecodeError) def testStrictUnicodeUnpack(): - unpackb(packb(b'abc\xeddef'), encoding='utf-8', use_list=1) + with raises(UnicodeDecodeError): + unpackb(packb(b'abc\xeddef'), encoding='utf-8', use_list=1) -@raises(UnicodeEncodeError) def testStrictUnicodePack(): - packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict') + with raises(UnicodeEncodeError): + packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict') def testIgnoreErrorsPack(): re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1) - assert_equal(re, six.u("abcdef")) + assert re == six.u("abcdef") -@raises(TypeError) def testNoEncoding(): - packb(six.u("abc"), encoding=None) + with raises(TypeError): + packb(six.u("abc"), encoding=None) def testDecodeBinary(): re = unpackb(packb("abc"), encoding=None, use_list=1) - assert_equal(re, b"abc") + assert re == b"abc" def testPackFloat(): - assert_equal(packb(1.0, use_single_float=True), b'\xca' + struct.pack('>f', 1.0)) - assert_equal(packb(1.0, use_single_float=False), b'\xcb' + struct.pack('>d', 1.0)) + assert packb(1.0, use_single_float=True) == b'\xca' + struct.pack('>f', 1.0) + assert packb(1.0, use_single_float=False) == b'\xcb' + struct.pack('>d', 1.0) def testArraySize(sizes=[0, 5, 50, 1000]): bio = six.BytesIO() @@ -151,10 +149,10 @@ class odict(dict): def test_odict(): seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)] od = odict(seq) - assert_equal(unpackb(packb(od), use_list=1), dict(seq)) + assert unpackb(packb(od), use_list=1) == dict(seq) def pair_hook(seq): return seq - assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1), seq) + assert unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1) == seq def test_pairlist(): @@ -163,8 +161,3 @@ def test_pairlist(): packed = packer.pack_map_pairs(pairlist) unpacked = unpackb(packed, object_pairs_hook=list) assert pairlist == unpacked - - - -if __name__ == '__main__': - main() diff --git a/test/test_seq.py b/test/test_seq.py index 72e935a..af719b0 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -2,9 +2,6 @@ # coding: utf-8 import six -from nose import main -from nose.tools import * - import io import msgpack @@ -38,13 +35,8 @@ def test_exceeding_unpacker_read_size(): read_count = 0 for idx, o in enumerate(unpacker): - assert_equal(type(o), bytes) - assert_equal(o, gen_binary_data(idx)) + assert type(o) == bytes + assert o == gen_binary_data(idx) read_count += 1 - assert_equal(read_count, NUMBER_OF_STRINGS) - - -if __name__ == '__main__': - main() - #test_exceeding_unpacker_read_size() + assert read_count == NUMBER_OF_STRINGS diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index f767726..fc1f712 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -4,7 +4,8 @@ import six from msgpack import Unpacker, BufferFull from msgpack.exceptions import OutOfData -import nose +from pytest import raises + def test_foobar(): unpacker = Unpacker(read_size=3, use_list=1) @@ -15,11 +16,8 @@ def test_foobar(): assert unpacker.unpack() == ord(b'b') assert unpacker.unpack() == ord(b'a') assert unpacker.unpack() == ord(b'r') - try: - o = unpacker.unpack() - assert 0, "should raise exception" - except OutOfData: - assert 1, "ok" + with raises(OutOfData): + unpacker.unpack() unpacker.feed(b'foo') unpacker.feed(b'bar') @@ -39,17 +37,16 @@ def test_foobar_skip(): unpacker.skip() assert unpacker.unpack() == ord(b'a') unpacker.skip() - try: - o = unpacker.unpack() - assert 0, "should raise exception" - except OutOfData: - assert 1, "ok" + with raises(OutOfData): + unpacker.unpack() def test_maxbuffersize(): - nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3) + with raises(ValueError): + Unpacker(read_size=5, max_buffer_size=3) unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1) unpacker.feed(b'fo') - nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob') + with raises(BufferFull): + unpacker.feed(b'ob') unpacker.feed(b'o') assert ord('f') == next(unpacker) unpacker.feed(b'b') @@ -73,5 +70,3 @@ def test_readbytes(): assert unpacker.unpack() == ord(b'a') assert unpacker.unpack() == ord(b'r') -if __name__ == '__main__': - nose.main() diff --git a/test/test_subtype.py b/test/test_subtype.py index 1dfd7da..6807508 100644 --- a/test/test_subtype.py +++ b/test/test_subtype.py @@ -1,8 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -from nose import main -from nose.tools import * from msgpack import packb, unpackb from collections import namedtuple @@ -18,10 +16,6 @@ class MyTuple(tuple): MyNamedTuple = namedtuple('MyNamedTuple', 'x y') def test_types(): - assert_equal(packb(dict()), packb(MyDict())) - assert_equal(packb(list()), packb(MyList())) - assert_equal(packb(MyNamedTuple(1,2)), packb((1,2))) - - -if __name__ == '__main__': - main() + assert packb(MyDict()) == packb(dict()) + assert packb(MyList()) == packb(list()) + assert packb(MyNamedTuple(1, 2)) == packb((1, 2)) diff --git a/test/test_unpack_raw.py b/test/test_unpack_raw.py index 15d9c93..9f3784c 100644 --- a/test/test_unpack_raw.py +++ b/test/test_unpack_raw.py @@ -1,7 +1,5 @@ """Tests for cases where the user seeks to obtain packed msgpack objects""" -from nose import main -from nose.tools import * import six from msgpack import Unpacker, packb @@ -10,14 +8,14 @@ def test_write_bytes(): unpacker = Unpacker() unpacker.feed(b'abc') f = six.BytesIO() - assert_equal(unpacker.unpack(f.write), ord('a')) - assert_equal(f.getvalue(), b'a') + assert unpacker.unpack(f.write) == ord('a') + assert f.getvalue() == b'a' f = six.BytesIO() assert unpacker.skip(f.write) is None - assert_equal(f.getvalue(), b'b') + assert f.getvalue() == b'b' f = six.BytesIO() assert unpacker.skip() is None - assert_equal(f.getvalue(), b'') + assert f.getvalue() == b'' def test_write_bytes_multi_buffer(): @@ -27,8 +25,5 @@ def test_write_bytes_multi_buffer(): f = six.BytesIO() unpacked = unpacker.unpack(f.write) - assert_equal(unpacked, long_val) - assert_equal(f.getvalue(), expected) - -if __name__ == '__main__': - main() + assert unpacked == long_val + assert f.getvalue() == expected From 97a9f3f05ccff6ba548172a7c07cdfe80ae2a12b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 29 Dec 2012 11:27:28 +0900 Subject: [PATCH 0649/1172] Switching to py.test --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 214c4c4..1951352 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = py26,py27,py32,py33 [testenv] deps= - nose + pytest six -commands=nosetests -w test +commands=py.test test From 5f55e4c6dbc3ec723bea5b9fead2e36224e70b81 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 29 Dec 2012 11:28:28 +0900 Subject: [PATCH 0650/1172] Switchng to py.test --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 245c09c..875a3f6 100644 --- a/Makefile +++ b/Makefile @@ -9,4 +9,4 @@ python3: python3 setup.py build sdist test: - nosetests test + py.test test From 6a28b28c6314158ee099c95d452406278cdb93b0 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Mon, 28 Jan 2013 12:27:24 +0100 Subject: [PATCH 0651/1172] Add pure Python fallback module Signed-off-by: Bas Westerbaan --- msgpack/__init__.py | 8 +- msgpack/fallback.py | 476 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 482 insertions(+), 2 deletions(-) create mode 100644 msgpack/fallback.py diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 0bfe6fe..49a32d9 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,8 +1,12 @@ # coding: utf-8 from msgpack._version import version from msgpack.exceptions import * -from msgpack._packer import pack, packb, Packer -from msgpack._unpacker import unpack, unpackb, Unpacker + +try: + from msgpack._packer import pack, packb, Packer + from msgpack._unpacker import unpack, unpackb, Unpacker +except ImportError: + from msgpack.fallback import pack, packb, Packer, unpack, unpackb, Unpacker # alias for compatibility to simplejson/marshal/pickle. load = unpack diff --git a/msgpack/fallback.py b/msgpack/fallback.py new file mode 100644 index 0000000..2ca3a8f --- /dev/null +++ b/msgpack/fallback.py @@ -0,0 +1,476 @@ +# Fallback pure Python implementation of msgpack + +import sys +import array +import struct + +try: + import cStringIO as StringIO +except ImportError: + import StringIO + +from msgpack.exceptions import ( + BufferFull, + OutOfData, + UnpackValueError, + PackValueError, + ExtraData) + +EX_SKIP = 0 +EX_CONSTRUCT = 1 +EX_READ_ARRAY_HEADER = 2 +EX_READ_MAP_HEADER = 3 + +TYPE_IMMEDIATE = 0 +TYPE_ARRAY = 1 +TYPE_MAP = 2 +TYPE_RAW = 3 + +DEFAULT_RECURSE_LIMIT=511 + +def pack(o, stream, default=None, encoding='utf-8', unicode_errors='strict'): + """ Pack object `o` and write it to `stream` """ + packer = Packer(default=default, encoding=encoding, + unicode_errors=unicode_errors) + stream.write(packer.pack(o)) + +def packb(o, default=None, encoding='utf-8', unicode_errors='struct', + use_single_float=False): + """ Pack object `o` and return packed bytes """ + packer = Packer(default=default, + encoding=encoding, + unicode_errors=unicode_errors, + use_single_float=use_single_float) + return packer.pack(o) + +def unpack(stream, object_hook=None, list_hook=None, use_list=True, + encoding=None, unicode_errors='strict', + object_pairs_hook=None): + """ Unpack an object from `stream`. + + Raises `ExtraData` when `stream` has extra bytes. """ + unpacker = Unpacker(stream, object_hook=object_hook, list_hook=list_hook, + use_list=use_list, encoding=encoding, unicode_errors=unicode_errors, + object_pairs_hook=object_pairs_hook) + ret = unpacker._fb_unpack() + unpacker._fb_check_for_extradata() + return ret + +def unpackb(packed, object_hook=None, list_hook=None, use_list=True, + encoding=None, unicode_errors='strict', + object_pairs_hook=None): + """ Unpack an object from `packed`. + + Raises `ExtraData` when `packed` contains extra bytes. """ + unpacker = Unpacker(None, object_hook=object_hook, list_hook=list_hook, + use_list=use_list, encoding=encoding, unicode_errors=unicode_errors, + object_pairs_hook=object_pairs_hook) + unpacker.feed(packed) + ret = unpacker._fb_unpack() + unpacker._fb_check_for_extradata() + return ret + +class Unpacker(object): + """ + Streaming unpacker. + + `file_like` is a file-like object having a `.read(n)` method. + When `Unpacker` is initialized with a `file_like`, `.feed()` is not + usable. + + `read_size` is used for `file_like.read(read_size)`. + + If `use_list` is True (default), msgpack lists are deserialized to Python + lists. Otherwise they are deserialized to tuples. + + `object_hook` is the same as in simplejson. If it is not None, it should + be callable and Unpacker calls it with a dict argument after deserializing + a map. + + `object_pairs_hook` is the same as in simplejson. If it is not None, it + should be callable and Unpacker calls it with a list of key-value pairs + after deserializing a map. + + `encoding` is the encoding used for decoding msgpack bytes. If it is + None (default), msgpack bytes are deserialized to Python bytes. + + `unicode_errors` is used for decoding bytes. + + `max_buffer_size` limits the buffer size. 0 means INT_MAX (default). + + Raises `BufferFull` exception when it is unsufficient. + + You should set this parameter when unpacking data from an untrustred source. + + example of streaming deserialization from file-like object:: + + unpacker = Unpacker(file_like) + for o in unpacker: + do_something(o) + + example of streaming deserialization from socket:: + + unpacker = Unapcker() + while 1: + buf = sock.recv(1024*2) + if not buf: + break + unpacker.feed(buf) + for o in unpacker: + do_something(o) + """ + + def __init__(self, file_like=None, read_size=0, use_list=True, + object_hook=None, object_pairs_hook=None, list_hook=None, + encoding=None, unicode_errors='strict', max_buffer_size=0): + if file_like is None: + self._fb_feeding = True + else: + if not callable(file_like.read): + raise ValueError("`file_like.read` must be callable") + self.file_like = file_like + self._fb_feeding = False + self._fb_buffers = [] + self._fb_buf_o = 0 + self._fb_buf_i = 0 + self._fb_buf_n = 0 + self.max_buffer_size = (sys.maxint if max_buffer_size == 0 + else max_buffer_size) + self.read_size = (read_size if read_size != 0 + else min(self.max_buffer_size, 2048)) + if read_size > self.max_buffer_size: + raise ValueError("read_size must be smaller than max_buffer_size") + self.encoding = encoding + self.unicode_errors = unicode_errors + self.use_list = use_list + self.list_hook = list_hook + self.object_hook = object_hook + self.object_pairs_hook = object_pairs_hook + + if list_hook is not None and not callable(list_hook): + raise ValueError('`list_hook` is not callable') + if object_hook is not None and not callable(object_hook): + raise ValueError('`object_hook` is not callable') + if object_pairs_hook is not None and not callable(object_pairs_hook): + raise ValueError('`object_pairs_hook` is not callable') + if object_hook is not None and object_pairs_hook is not None: + raise ValueError("object_pairs_hook and object_hook are mutually "+ + "exclusive") + + def feed(self, next_bytes): + if isinstance(next_bytes, array.array): + next_bytes = next_bytes.tostring() + assert self._fb_feeding + if self._fb_buf_n + len(next_bytes) > self.max_buffer_size: + raise BufferFull + self._fb_buf_n += len(next_bytes) + self._fb_buffers.append(next_bytes) + + def _fb_consume(self): + self._fb_buffers = self._fb_buffers[self._fb_buf_i:] + if self._fb_buffers: + self._fb_buffers[0] = self._fb_buffers[0][self._fb_buf_o:] + self._fb_buf_o = 0 + self._fb_buf_i = 0 + self._fb_buf_n = sum(map(len, self._fb_buffers)) + + def _fb_check_for_extradata(self): + if self._fb_buf_i != len(self._fb_buffers): + raise ExtraData + if self._fb_feeding: + return + if not self.file_like: + return + if not self.file_like.read(1): + raise ExtraData + + def __iter__(self): + return self + + def next(self): + try: + ret = self._fb_unpack(None, None) + self._fb_consume() + return ret + except OutOfData: + raise StopIteration + + def read_bytes(self, n): + return self._fb_read(n) + + def _fb_read(self, n, write_bytes=None): + ret = '' + while len(ret) != n: + if self._fb_buf_i == len(self._fb_buffers): + if self._fb_feeding: + break + tmp = self.file_like.read(self.read_size) + if not tmp: + break + self._fb_buffers.append(tmp) + continue + sliced = n - len(ret) + ret += self._fb_buffers[self._fb_buf_i][ + self._fb_buf_o:self._fb_buf_o + sliced] + self._fb_buf_o += sliced + if self._fb_buf_o >= len(self._fb_buffers[self._fb_buf_i]): + self._fb_buf_o = 0 + self._fb_buf_i += 1 + if len(ret) != n: + raise OutOfData + if write_bytes is not None: + write_bytes(ret) + return ret + + def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None): + typ = TYPE_IMMEDIATE + b = ord(self._fb_read(1, write_bytes)) + if b & 0b10000000 == 0: + obj = b + elif b & 0b11100000 == 0b11100000: + obj = struct.unpack("b", chr(b))[0] + elif b & 0b11100000 == 0b10100000: + n = b & 0b00011111 + obj = struct.unpack("%ds" % n, self._fb_read(n, write_bytes))[0] + typ = TYPE_RAW + elif b & 0b11110000 == 0b10010000: + n = b & 0b00001111 + typ = TYPE_ARRAY + elif b & 0b11110000 == 0b10000000: + n = b & 0b00001111 + typ = TYPE_MAP + elif b == 0xc0: + obj = None + elif b == 0xc2: + obj = False + elif b == 0xc3: + obj = True + elif b == 0xca: + obj = struct.unpack(">f", self._fb_read(4, write_bytes))[0] + elif b == 0xcb: + obj = struct.unpack(">d", self._fb_read(8, write_bytes))[0] + elif b == 0xcc: + obj = struct.unpack("B", self._fb_read(1, write_bytes))[0] + elif b == 0xcd: + obj = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + elif b == 0xce: + obj = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + elif b == 0xcf: + obj = struct.unpack(">Q", self._fb_read(8, write_bytes))[0] + elif b == 0xd0: + obj = struct.unpack("b", self._fb_read(1, write_bytes))[0] + elif b == 0xd1: + obj = struct.unpack(">h", self._fb_read(2, write_bytes))[0] + elif b == 0xd2: + obj = struct.unpack(">i", self._fb_read(4, write_bytes))[0] + elif b == 0xd3: + obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0] + elif b == 0xda: + n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + obj = struct.unpack("%ds" % n, self._fb_read(n, write_bytes))[0] + typ = TYPE_RAW + elif b == 0xdb: + n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + obj = struct.unpack("%ds" % n, self._fb_read(n, write_bytes))[0] + typ = TYPE_RAW + elif b == 0xdc: + n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + typ = TYPE_ARRAY + elif b == 0xdd: + n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + typ = TYPE_ARRAY + elif b == 0xde: + n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + typ = TYPE_MAP + elif b == 0xdf: + n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + typ = TYPE_MAP + else: + raise UnpackValueError("Unknown header: 0x%x" % b) + if execute == EX_READ_ARRAY_HEADER: + if typ != TYPE_ARRAY: + raise UnpackValueError("Expected array") + return n + if execute == EX_READ_MAP_HEADER: + if typ != TYPE_MAP: + raise UnpackValueError("Expected map") + return n + # TODO should we eliminate the recursion? + if typ == TYPE_ARRAY: + if execute == EX_SKIP: + for i in xrange(n): + # TODO check whether we need to call `list_hook` + self._fb_unpack(EX_SKIP, write_bytes) + return + ret = [] + for i in xrange(n): + ret.append(self._fb_unpack(EX_CONSTRUCT, write_bytes)) + if self.list_hook is not None: + ret = self.list_hook(ret) + # TODO is the interaction between `list_hook` and `use_list` ok? + return ret if self.use_list else tuple(ret) + if typ == TYPE_MAP: + if execute == EX_SKIP: + for i in xrange(n): + # TODO check whether we need to call hooks + self._fb_unpack(EX_SKIP, write_bytes) + self._fb_unpack(EX_SKIP, write_bytes) + return + ret = [] + for i in xrange(n): + ret.append((self._fb_unpack(EX_CONSTRUCT, write_bytes), + self._fb_unpack(EX_CONSTRUCT, write_bytes))) + if self.object_pairs_hook is not None: + ret = self.object_pairs_hook(ret) + else: + ret = dict(ret) + if self.object_hook is not None: + ret = self.object_hook(ret) + return ret + if execute == EX_SKIP: + return + if typ == TYPE_RAW: + if self.encoding is not None: + obj = obj.decode(self.encoding, self.unicode_errors) + return obj + assert typ == TYPE_IMMEDIATE + return obj + + def skip(self, write_bytes=None): + self._fb_unpack(EX_SKIP, write_bytes) + self._fb_consume() + + def unpack(self, write_bytes=None): + ret = self._fb_unpack(EX_CONSTRUCT, write_bytes) + self._fb_consume() + return ret + + def read_array_header(self, write_bytes=None): + ret = self._fb_unpack(EX_READ_ARRAY_HEADER, write_bytes) + self._fb_consume() + return ret + + def read_map_header(self, write_bytes=None): + ret = self._fb_unpack(EX_READ_MAP_HEADER, write_bytes) + self._fb_consume() + return ret + +class Packer(object): + def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', + use_single_float=False, autoreset=True): + self.use_float = use_single_float + self.autoreset = autoreset + self.encoding = encoding + self.unicode_errors = unicode_errors + self.buffer = StringIO.StringIO() + if default is not None: + if not callable(default): + raise TypeError("default must be callable") + self._default = default + def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT): + if nest_limit < 0: + raise PackValueError("recursion limit exceeded") + if obj is None: + return self.buffer.write(chr(0xc0)) + if isinstance(obj, bool): + if obj: + return self.buffer.write(chr(0xc3)) + return self.buffer.write(chr(0xc2)) + if isinstance(obj, int) or isinstance(obj, long): + if 0 <= obj < 0x80: + return self.buffer.write(struct.pack("B", obj)) + if -0x20 <= obj < 0: + return self.buffer.write(struct.pack("b", obj)) + if 0x80 <= obj <= 0xff: + return self.buffer.write(struct.pack("BB", 0xcc, obj)) + if -0x80 <= obj < 0: + return self.buffer.write(struct.pack(">Bb", 0xd0, obj)) + if 0xff < obj <= 0xffff: + return self.buffer.write(struct.pack(">BH", 0xcd, obj)) + if -0x8000 <= obj < -0x80: + return self.buffer.write(struct.pack(">Bh", 0xd1, obj)) + if 0xffff < obj <= 0xffffffff: + return self.buffer.write(struct.pack(">BI", 0xce, obj)) + if -0x80000000 <= obj < -0x8000: + return self.buffer.write(struct.pack(">Bi", 0xd2, obj)) + if 0xffffffff < obj <= 0xffffffffffffffff: + return self.buffer.write(struct.pack(">BQ", 0xcf, obj)) + if -0x8000000000000000 <= obj < -0x80000000: + return self.buffer.write(struct.pack(">Bq", 0xd3, obj)) + raise PackValueError("Integer value out of range") + if isinstance(obj, str) or isinstance(obj, unicode): + if isinstance(obj, unicode): + obj = obj.encode(self.encoding, self.unicode_errors) + n = len(obj) + if n <= 0x1f: + return self.buffer.write(chr(0xa0 + n) + obj) + if n <= 0xffff: + return self.buffer.write(struct.pack(">BH%ds" % n,0xda, n, obj)) + if n <= 0xffffffff: + return self.buffer.write(struct.pack(">BI%ds" % n,0xdb, n, obj)) + raise PackValueError("String is too large") + if isinstance(obj, float): + if self.use_float: + return self.buffer.write(struct.pack(">Bf", 0xca, obj)) + return self.buffer.write(struct.pack(">Bd", 0xcb, obj)) + if isinstance(obj, list) or isinstance(obj, tuple): + n = len(obj) + self._fb_pack_array_header(n) + for i in xrange(n): + self._pack(obj[i], nest_limit - 1) + return + if isinstance(obj, dict): + return self._fb_pack_map_pairs(len(obj), obj.iteritems(), + nest_limit - 1) + if self._default is not None: + return self._pack(self._default(obj), nest_limit - 1) + raise TypeError("Cannot serialize %r" % obj) + def pack(self, obj): + self._pack(obj) + ret = self.buffer.getvalue() + if self.autoreset: + self.buffer = StringIO.StringIO() + return ret + def pack_map_pairs(self, pairs): + self._fb_pack_map_pairs(len(pairs), pairs) + ret = self.buffer.getvalue() + if self.autoreset: + self.buffer = StringIO.StringIO() + return ret + def pack_array_header(self, n): + self._fb_pack_array_header(n) + ret = self.buffer.getvalue() + if self.autoreset: + self.buffer = StringIO.StringIO() + return ret + def pack_map_header(self, n): + self._fb_pack_map_header(n) + ret = self.buffer.getvalue() + if self.autoreset: + self.buffer = StringIO.StringIO() + return ret + def _fb_pack_array_header(self, n): + if n <= 0x0f: + return self.buffer.write(chr(0x90 + n)) + if n <= 0xffff: + return self.buffer.write(struct.pack(">BH", 0xdc, n)) + if n <= 0xffffffff: + return self.buffer.write(struct.pack(">BI", 0xdd, n)) + raise PackValueError("Array is too large") + def _fb_pack_map_header(self, n): + if n <= 0x0f: + return self.buffer.write(chr(0x80 + n)) + if n <= 0xffff: + return self.buffer.write(struct.pack(">BH", 0xde, n)) + if n <= 0xffffffff: + return self.buffer.write(struct.pack(">BI", 0xdf, n)) + raise PackValueError("Dict is too large") + def _fb_pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT): + self._fb_pack_map_header(n) + for (k, v) in pairs: + self._pack(k, nest_limit - 1) + self._pack(v, nest_limit - 1) + def bytes(self): + return self.buffer.getvalue() + def reset(self): + self.buffer = StringIO.StringIO() From 2627b6ae9f0606b388a3a3ec0110d1fdb33d082e Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Mon, 28 Jan 2013 12:27:46 +0100 Subject: [PATCH 0652/1172] setup: automatically fallback to pure Python module Signed-off-by: Bas Westerbaan --- setup.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index ac3eeb5..3d7fc04 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,9 @@ from setuptools import setup, Extension from distutils.command.build_ext import build_ext +class NoCython(Exception): + pass + try: import Cython.Compiler.Main as cython_compiler have_cython = True @@ -24,10 +27,7 @@ def ensure_source(src): if not os.path.exists(src): if not have_cython: - raise Exception("""\ -Cython is required for building extension from checkout. -Install Cython >= 0.16 or install msgpack from PyPI. -""") + raise NoCython cythonize(pyx) elif (os.path.exists(pyx) and os.stat(src).st_mtime < os.stat(pyx).st_mtime and @@ -38,7 +38,14 @@ Install Cython >= 0.16 or install msgpack from PyPI. class BuildExt(build_ext): def build_extension(self, ext): - ext.sources = list(map(ensure_source, ext.sources)) + try: + ext.sources = list(map(ensure_source, ext.sources)) + except NoCython: + print "WARNING" + print "Cython is required for building extension from checkout." + print "Install Cython >= 0.16 or install msgpack from PyPI." + print "Falling back to pure Python implementation." + return return build_ext.build_extension(self, ext) From 69ba3c9bf9737df1ade7832986b0fa6f1659b04d Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Mon, 28 Jan 2013 13:30:32 +0100 Subject: [PATCH 0653/1172] fallback: use __pypy__.builders.StringBuilder when available This increases performance *a lot* on PyPy. Signed-off-by: Bas Westerbaan --- msgpack/fallback.py | 46 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 2ca3a8f..220e0fe 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -4,10 +4,28 @@ import sys import array import struct -try: - import cStringIO as StringIO -except ImportError: - import StringIO +if hasattr(sys, 'pypy_version_info'): + # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own + # StringBuilder is fastest. + from __pypy__.builders import StringBuilder + USING_STRINGBUILDER = True + class StringIO(object): + def __init__(self, s=''): + if s: + self.builder = StringBuilder(len(s)) + self.builder.append(s) + else: + self.builder = StringBuilder() + def write(self, s): + self.builder.append(s) + def getvalue(self): + return self.builder.build() +else: + USING_STRINGBUILDER = False + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO from msgpack.exceptions import ( BufferFull, @@ -362,7 +380,7 @@ class Packer(object): self.autoreset = autoreset self.encoding = encoding self.unicode_errors = unicode_errors - self.buffer = StringIO.StringIO() + self.buffer = StringIO() if default is not None: if not callable(default): raise TypeError("default must be callable") @@ -429,25 +447,33 @@ class Packer(object): self._pack(obj) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = StringIO.StringIO() + self.buffer = StringIO() + elif USING_STRINGBUILDER: + self.buffer = StringIO(ret) return ret def pack_map_pairs(self, pairs): self._fb_pack_map_pairs(len(pairs), pairs) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = StringIO.StringIO() + self.buffer = StringIO() + elif USING_STRINGBUILDER: + self.buffer = StringIO(ret) return ret def pack_array_header(self, n): self._fb_pack_array_header(n) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = StringIO.StringIO() + self.buffer = StringIO() + elif USING_STRINGBUILDER: + self.buffer = StringIO(ret) return ret def pack_map_header(self, n): self._fb_pack_map_header(n) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = StringIO.StringIO() + self.buffer = StringIO() + elif USING_STRINGBUILDER: + self.buffer = StringIO(ret) return ret def _fb_pack_array_header(self, n): if n <= 0x0f: @@ -473,4 +499,4 @@ class Packer(object): def bytes(self): return self.buffer.getvalue() def reset(self): - self.buffer = StringIO.StringIO() + self.buffer = StringIO() From 6fa0f46a122c4f6be35415a3e65dcdc542fd3acd Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Mon, 28 Jan 2013 14:32:01 +0100 Subject: [PATCH 0654/1172] setup: remove Python 2 only syntax --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 3d7fc04..d4808d6 100644 --- a/setup.py +++ b/setup.py @@ -41,10 +41,10 @@ class BuildExt(build_ext): try: ext.sources = list(map(ensure_source, ext.sources)) except NoCython: - print "WARNING" - print "Cython is required for building extension from checkout." - print "Install Cython >= 0.16 or install msgpack from PyPI." - print "Falling back to pure Python implementation." + print("WARNING") + print("Cython is required for building extension from checkout.") + print("Install Cython >= 0.16 or install msgpack from PyPI.") + print("Falling back to pure Python implementation.") return return build_ext.build_extension(self, ext) From b940802032adc16b074b1e29bed72825580c5c9f Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Mon, 28 Jan 2013 22:29:23 +0100 Subject: [PATCH 0655/1172] fallback: two fixes for raising ExtraData --- msgpack/fallback.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 220e0fe..392f7eb 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -71,7 +71,8 @@ def unpack(stream, object_hook=None, list_hook=None, use_list=True, use_list=use_list, encoding=encoding, unicode_errors=unicode_errors, object_pairs_hook=object_pairs_hook) ret = unpacker._fb_unpack() - unpacker._fb_check_for_extradata() + if unpacker._fb_got_extradata(): + raise ExtraData(ret, unpacker._fb_get_extradata()) return ret def unpackb(packed, object_hook=None, list_hook=None, use_list=True, @@ -85,7 +86,8 @@ def unpackb(packed, object_hook=None, list_hook=None, use_list=True, object_pairs_hook=object_pairs_hook) unpacker.feed(packed) ret = unpacker._fb_unpack() - unpacker._fb_check_for_extradata() + if unpacker._fb_got_extradata(): + raise ExtraData(ret, unpacker._fb_get_extradata()) return ret class Unpacker(object): @@ -192,15 +194,16 @@ class Unpacker(object): self._fb_buf_i = 0 self._fb_buf_n = sum(map(len, self._fb_buffers)) - def _fb_check_for_extradata(self): + def _fb_got_extradata(self): if self._fb_buf_i != len(self._fb_buffers): - raise ExtraData + return True if self._fb_feeding: - return + return False if not self.file_like: - return - if not self.file_like.read(1): - raise ExtraData + return False + if self.file_like.read(1): + return True + return False def __iter__(self): return self @@ -216,6 +219,12 @@ class Unpacker(object): def read_bytes(self, n): return self._fb_read(n) + def _fb_get_extradata(self): + bufs = self._fb_buffers[self._fb_buf_i:] + if bufs: + bufs[0] = bufs[0][self._fb_buf_o:] + return ''.join(bufs) + def _fb_read(self, n, write_bytes=None): ret = '' while len(ret) != n: From af9c9ca2c9cae01cf603da90530b9ba396007e5b Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Tue, 29 Jan 2013 02:01:34 +0100 Subject: [PATCH 0656/1172] fallback: performance: write(a+b) -> write(a); write(b) --- msgpack/fallback.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 392f7eb..d6fd533 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -430,7 +430,8 @@ class Packer(object): obj = obj.encode(self.encoding, self.unicode_errors) n = len(obj) if n <= 0x1f: - return self.buffer.write(chr(0xa0 + n) + obj) + self.buffer.write(chr(0xa0 + n)) + return self.buffer.write(obj) if n <= 0xffff: return self.buffer.write(struct.pack(">BH%ds" % n,0xda, n, obj)) if n <= 0xffffffff: From 94925acb124998aaf9438e7c8152fbfbb7d5d8a8 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Tue, 29 Jan 2013 02:15:29 +0100 Subject: [PATCH 0657/1172] fallback: do not use dynamic format strings for struct.(un)pack Increases performance on PyPy. --- msgpack/fallback.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index d6fd533..fe2b1ac 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -258,7 +258,7 @@ class Unpacker(object): obj = struct.unpack("b", chr(b))[0] elif b & 0b11100000 == 0b10100000: n = b & 0b00011111 - obj = struct.unpack("%ds" % n, self._fb_read(n, write_bytes))[0] + obj = self._fb_read(n, write_bytes) typ = TYPE_RAW elif b & 0b11110000 == 0b10010000: n = b & 0b00001111 @@ -294,11 +294,11 @@ class Unpacker(object): obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0] elif b == 0xda: n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] - obj = struct.unpack("%ds" % n, self._fb_read(n, write_bytes))[0] + obj = self._fb_read(n, write_bytes) typ = TYPE_RAW elif b == 0xdb: n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] - obj = struct.unpack("%ds" % n, self._fb_read(n, write_bytes))[0] + obj = self._fb_read(n, write_bytes) typ = TYPE_RAW elif b == 0xdc: n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] @@ -433,9 +433,11 @@ class Packer(object): self.buffer.write(chr(0xa0 + n)) return self.buffer.write(obj) if n <= 0xffff: - return self.buffer.write(struct.pack(">BH%ds" % n,0xda, n, obj)) + self.buffer.write(struct.pack(">BH", 0xda, n)) + return self.buffer.write(obj) if n <= 0xffffffff: - return self.buffer.write(struct.pack(">BI%ds" % n,0xdb, n, obj)) + self.buffer.write(struct.pack(">BI", 0xdb, n)) + return self.buffer.write(obj) raise PackValueError("String is too large") if isinstance(obj, float): if self.use_float: From fb81f80d14613bd2ac3e63029a47bb0512c25dd5 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Tue, 29 Jan 2013 02:47:41 +0100 Subject: [PATCH 0658/1172] fallback: bugfix in next() --- msgpack/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index fe2b1ac..a866ff1 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -210,7 +210,7 @@ class Unpacker(object): def next(self): try: - ret = self._fb_unpack(None, None) + ret = self._fb_unpack(EX_CONSTRUCT, None) self._fb_consume() return ret except OutOfData: From d2f549a47094b2a29cc94bc50029ebdc85508861 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Tue, 29 Jan 2013 02:58:26 +0100 Subject: [PATCH 0659/1172] fallback: add actual rollback and add a testcase for it Signed-off-by: Bas Westerbaan --- msgpack/fallback.py | 21 +++++++++++++-------- test/test_sequnpack.py | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index a866ff1..2f83a20 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -208,17 +208,13 @@ class Unpacker(object): def __iter__(self): return self - def next(self): - try: - ret = self._fb_unpack(EX_CONSTRUCT, None) - self._fb_consume() - return ret - except OutOfData: - raise StopIteration - def read_bytes(self, n): return self._fb_read(n) + def _fb_rollback(self): + self._fb_buf_i = 0 + self._fb_buf_o = 0 + def _fb_get_extradata(self): bufs = self._fb_buffers[self._fb_buf_i:] if bufs: @@ -244,6 +240,7 @@ class Unpacker(object): self._fb_buf_o = 0 self._fb_buf_i += 1 if len(ret) != n: + self._fb_rollback() raise OutOfData if write_bytes is not None: write_bytes(ret) @@ -363,6 +360,14 @@ class Unpacker(object): assert typ == TYPE_IMMEDIATE return obj + def next(self): + try: + ret = self._fb_unpack(EX_CONSTRUCT, None) + self._fb_consume() + return ret + except OutOfData: + raise StopIteration + def skip(self, write_bytes=None): self._fb_unpack(EX_SKIP, write_bytes) self._fb_consume() diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index fc1f712..1da383c 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -7,6 +7,21 @@ from msgpack.exceptions import OutOfData from pytest import raises +def test_partialdata(): + unpacker = Unpacker() + unpacker.feed(b'\xa5') + with raises(StopIteration): next(iter(unpacker)) + unpacker.feed(b'h') + with raises(StopIteration): next(iter(unpacker)) + unpacker.feed(b'a') + with raises(StopIteration): next(iter(unpacker)) + unpacker.feed(b'l') + with raises(StopIteration): next(iter(unpacker)) + unpacker.feed(b'l') + with raises(StopIteration): next(iter(unpacker)) + unpacker.feed(b'o') + assert next(iter(unpacker)) == 'hallo' + def test_foobar(): unpacker = Unpacker(read_size=3, use_list=1) unpacker.feed(b'foobar') From b9e9199eea3a1654dca3e6faf6530d438788ad77 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Tue, 29 Jan 2013 03:03:13 +0100 Subject: [PATCH 0660/1172] fallback: python3 bugfix for new testcase of d2f549a4 Signed-off-by: Bas Westerbaan --- test/test_sequnpack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 1da383c..9db14ca 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -20,7 +20,7 @@ def test_partialdata(): unpacker.feed(b'l') with raises(StopIteration): next(iter(unpacker)) unpacker.feed(b'o') - assert next(iter(unpacker)) == 'hallo' + assert next(iter(unpacker)) == b'hallo' def test_foobar(): unpacker = Unpacker(read_size=3, use_list=1) From 770fed6b7f1ce1f6e9c20881897a417680c8b79a Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Tue, 29 Jan 2013 03:39:46 +0100 Subject: [PATCH 0661/1172] fallback: Use mmap objects instead of strings to unpack Signed-off-by: Bas Westerbaan --- msgpack/fallback.py | 80 +++++++++++++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 29 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 2f83a20..c0fcc18 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -1,15 +1,27 @@ # Fallback pure Python implementation of msgpack +# +# Easy imports +# import sys import array import struct +# +# Tricky imports +# +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +# We will use wStringIO for buffering the writes for packing. +# Normally, we will use cStringIO.StringIO. +# On PyPy we will use PyPy's own StringBuilder. if hasattr(sys, 'pypy_version_info'): - # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own - # StringBuilder is fastest. from __pypy__.builders import StringBuilder USING_STRINGBUILDER = True - class StringIO(object): + class wStringIO(object): def __init__(self, s=''): if s: self.builder = StringBuilder(len(s)) @@ -22,10 +34,18 @@ if hasattr(sys, 'pypy_version_info'): return self.builder.build() else: USING_STRINGBUILDER = False - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO + wStringIO = StringIO + +# We will use rStringIO for unpacking. +# Normally, this is a mmap. A normal StringIO is not a drop-in replacement --- +# it misses the __len__ operation. +# TODO add fallback for when mmap is unavailable +import mmap +def rStringIO(s): + m = mmap.mmap(-1, len(s)) + m.write(s) + m.seek(0) + return m from msgpack.exceptions import ( BufferFull, @@ -184,13 +204,13 @@ class Unpacker(object): if self._fb_buf_n + len(next_bytes) > self.max_buffer_size: raise BufferFull self._fb_buf_n += len(next_bytes) - self._fb_buffers.append(next_bytes) + self._fb_buffers.append(rStringIO(next_bytes)) def _fb_consume(self): self._fb_buffers = self._fb_buffers[self._fb_buf_i:] if self._fb_buffers: - self._fb_buffers[0] = self._fb_buffers[0][self._fb_buf_o:] - self._fb_buf_o = 0 + self._fb_buffers[0] = rStringIO(self._fb_buffers[0][ + self._fb_buffers[0].tell():]) self._fb_buf_i = 0 self._fb_buf_n = sum(map(len, self._fb_buffers)) @@ -212,16 +232,20 @@ class Unpacker(object): return self._fb_read(n) def _fb_rollback(self): + for buf in self._fb_buffers: + buf.seek(0) self._fb_buf_i = 0 - self._fb_buf_o = 0 def _fb_get_extradata(self): bufs = self._fb_buffers[self._fb_buf_i:] if bufs: - bufs[0] = bufs[0][self._fb_buf_o:] - return ''.join(bufs) + bufs[0] = rStringIO(bufs[0][bufs[0].tell():]) + return ''.join([buf[:] for buf in bufs]) def _fb_read(self, n, write_bytes=None): + if (write_bytes is None and self._fb_buf_i < len(self._fb_buffers) + and self._fb_buffers[0].tell() + n < len(self._fb_buffers[0])): + return self._fb_buffers[0].read(n) ret = '' while len(ret) != n: if self._fb_buf_i == len(self._fb_buffers): @@ -230,14 +254,12 @@ class Unpacker(object): tmp = self.file_like.read(self.read_size) if not tmp: break - self._fb_buffers.append(tmp) + self._fb_buffers.append(rStringIO(tmp)) continue sliced = n - len(ret) - ret += self._fb_buffers[self._fb_buf_i][ - self._fb_buf_o:self._fb_buf_o + sliced] - self._fb_buf_o += sliced - if self._fb_buf_o >= len(self._fb_buffers[self._fb_buf_i]): - self._fb_buf_o = 0 + ret += self._fb_buffers[self._fb_buf_i].read(sliced) + if (self._fb_buffers[self._fb_buf_i].tell() + == len(self._fb_buffers[self._fb_buf_i])): self._fb_buf_i += 1 if len(ret) != n: self._fb_rollback() @@ -394,7 +416,7 @@ class Packer(object): self.autoreset = autoreset self.encoding = encoding self.unicode_errors = unicode_errors - self.buffer = StringIO() + self.buffer = wStringIO() if default is not None: if not callable(default): raise TypeError("default must be callable") @@ -464,33 +486,33 @@ class Packer(object): self._pack(obj) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = StringIO() + self.buffer = wStringIO() elif USING_STRINGBUILDER: - self.buffer = StringIO(ret) + self.buffer = wStringIO(ret) return ret def pack_map_pairs(self, pairs): self._fb_pack_map_pairs(len(pairs), pairs) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = StringIO() + self.buffer = wStringIO() elif USING_STRINGBUILDER: - self.buffer = StringIO(ret) + self.buffer = wStringIO(ret) return ret def pack_array_header(self, n): self._fb_pack_array_header(n) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = StringIO() + self.buffer = wStringIO() elif USING_STRINGBUILDER: - self.buffer = StringIO(ret) + self.buffer = wStringIO(ret) return ret def pack_map_header(self, n): self._fb_pack_map_header(n) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = StringIO() + self.buffer = wStringIO() elif USING_STRINGBUILDER: - self.buffer = StringIO(ret) + self.buffer = wStringIO(ret) return ret def _fb_pack_array_header(self, n): if n <= 0x0f: @@ -516,4 +538,4 @@ class Packer(object): def bytes(self): return self.buffer.getvalue() def reset(self): - self.buffer = StringIO() + self.buffer = wStringIO() From d91a0d3d68bef38be667ba8e931e3f53cf90be6f Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Tue, 29 Jan 2013 03:45:17 +0100 Subject: [PATCH 0662/1172] Revert "fallback: Use mmap objects instead of strings to unpack" See next commit. This reverts commit 770fed6b7f1ce1f6e9c20881897a417680c8b79a. --- msgpack/fallback.py | 80 ++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 51 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index c0fcc18..2f83a20 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -1,27 +1,15 @@ # Fallback pure Python implementation of msgpack -# -# Easy imports -# import sys import array import struct -# -# Tricky imports -# -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - -# We will use wStringIO for buffering the writes for packing. -# Normally, we will use cStringIO.StringIO. -# On PyPy we will use PyPy's own StringBuilder. if hasattr(sys, 'pypy_version_info'): + # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own + # StringBuilder is fastest. from __pypy__.builders import StringBuilder USING_STRINGBUILDER = True - class wStringIO(object): + class StringIO(object): def __init__(self, s=''): if s: self.builder = StringBuilder(len(s)) @@ -34,18 +22,10 @@ if hasattr(sys, 'pypy_version_info'): return self.builder.build() else: USING_STRINGBUILDER = False - wStringIO = StringIO - -# We will use rStringIO for unpacking. -# Normally, this is a mmap. A normal StringIO is not a drop-in replacement --- -# it misses the __len__ operation. -# TODO add fallback for when mmap is unavailable -import mmap -def rStringIO(s): - m = mmap.mmap(-1, len(s)) - m.write(s) - m.seek(0) - return m + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO from msgpack.exceptions import ( BufferFull, @@ -204,13 +184,13 @@ class Unpacker(object): if self._fb_buf_n + len(next_bytes) > self.max_buffer_size: raise BufferFull self._fb_buf_n += len(next_bytes) - self._fb_buffers.append(rStringIO(next_bytes)) + self._fb_buffers.append(next_bytes) def _fb_consume(self): self._fb_buffers = self._fb_buffers[self._fb_buf_i:] if self._fb_buffers: - self._fb_buffers[0] = rStringIO(self._fb_buffers[0][ - self._fb_buffers[0].tell():]) + self._fb_buffers[0] = self._fb_buffers[0][self._fb_buf_o:] + self._fb_buf_o = 0 self._fb_buf_i = 0 self._fb_buf_n = sum(map(len, self._fb_buffers)) @@ -232,20 +212,16 @@ class Unpacker(object): return self._fb_read(n) def _fb_rollback(self): - for buf in self._fb_buffers: - buf.seek(0) self._fb_buf_i = 0 + self._fb_buf_o = 0 def _fb_get_extradata(self): bufs = self._fb_buffers[self._fb_buf_i:] if bufs: - bufs[0] = rStringIO(bufs[0][bufs[0].tell():]) - return ''.join([buf[:] for buf in bufs]) + bufs[0] = bufs[0][self._fb_buf_o:] + return ''.join(bufs) def _fb_read(self, n, write_bytes=None): - if (write_bytes is None and self._fb_buf_i < len(self._fb_buffers) - and self._fb_buffers[0].tell() + n < len(self._fb_buffers[0])): - return self._fb_buffers[0].read(n) ret = '' while len(ret) != n: if self._fb_buf_i == len(self._fb_buffers): @@ -254,12 +230,14 @@ class Unpacker(object): tmp = self.file_like.read(self.read_size) if not tmp: break - self._fb_buffers.append(rStringIO(tmp)) + self._fb_buffers.append(tmp) continue sliced = n - len(ret) - ret += self._fb_buffers[self._fb_buf_i].read(sliced) - if (self._fb_buffers[self._fb_buf_i].tell() - == len(self._fb_buffers[self._fb_buf_i])): + ret += self._fb_buffers[self._fb_buf_i][ + self._fb_buf_o:self._fb_buf_o + sliced] + self._fb_buf_o += sliced + if self._fb_buf_o >= len(self._fb_buffers[self._fb_buf_i]): + self._fb_buf_o = 0 self._fb_buf_i += 1 if len(ret) != n: self._fb_rollback() @@ -416,7 +394,7 @@ class Packer(object): self.autoreset = autoreset self.encoding = encoding self.unicode_errors = unicode_errors - self.buffer = wStringIO() + self.buffer = StringIO() if default is not None: if not callable(default): raise TypeError("default must be callable") @@ -486,33 +464,33 @@ class Packer(object): self._pack(obj) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = wStringIO() + self.buffer = StringIO() elif USING_STRINGBUILDER: - self.buffer = wStringIO(ret) + self.buffer = StringIO(ret) return ret def pack_map_pairs(self, pairs): self._fb_pack_map_pairs(len(pairs), pairs) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = wStringIO() + self.buffer = StringIO() elif USING_STRINGBUILDER: - self.buffer = wStringIO(ret) + self.buffer = StringIO(ret) return ret def pack_array_header(self, n): self._fb_pack_array_header(n) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = wStringIO() + self.buffer = StringIO() elif USING_STRINGBUILDER: - self.buffer = wStringIO(ret) + self.buffer = StringIO(ret) return ret def pack_map_header(self, n): self._fb_pack_map_header(n) ret = self.buffer.getvalue() if self.autoreset: - self.buffer = wStringIO() + self.buffer = StringIO() elif USING_STRINGBUILDER: - self.buffer = wStringIO(ret) + self.buffer = StringIO(ret) return ret def _fb_pack_array_header(self, n): if n <= 0x0f: @@ -538,4 +516,4 @@ class Packer(object): def bytes(self): return self.buffer.getvalue() def reset(self): - self.buffer = wStringIO() + self.buffer = StringIO() From 4cde7f080c16fd396d758930faf9ebf15b138047 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Tue, 29 Jan 2013 03:46:07 +0100 Subject: [PATCH 0663/1172] fallback: _fb_read: add fast-path --- msgpack/fallback.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 2f83a20..bd50b47 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -222,6 +222,11 @@ class Unpacker(object): return ''.join(bufs) def _fb_read(self, n, write_bytes=None): + if (write_bytes is None and self._fb_buf_i < len(self._fb_buffers) + and self._fb_buf_o + n < len(self._fb_buffers[self._fb_buf_i])): + self._fb_buf_o += n + return self._fb_buffers[self._fb_buf_i][ + self._fb_buf_o-n:self._fb_buf_o] ret = '' while len(ret) != n: if self._fb_buf_i == len(self._fb_buffers): From 328369e52e8ef41a8b4ec09e48870f8f3edb190c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 29 Jan 2013 14:33:37 +0900 Subject: [PATCH 0664/1172] pep8 friendly. --- msgpack/fallback.py | 46 ++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index bd50b47..3d733f4 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -1,4 +1,4 @@ -# Fallback pure Python implementation of msgpack +"""Fallback pure Python implementation of msgpack""" import sys import array @@ -49,11 +49,11 @@ DEFAULT_RECURSE_LIMIT=511 def pack(o, stream, default=None, encoding='utf-8', unicode_errors='strict'): """ Pack object `o` and write it to `stream` """ packer = Packer(default=default, encoding=encoding, - unicode_errors=unicode_errors) + unicode_errors=unicode_errors) stream.write(packer.pack(o)) def packb(o, default=None, encoding='utf-8', unicode_errors='struct', - use_single_float=False): + use_single_float=False): """ Pack object `o` and return packed bytes """ packer = Packer(default=default, encoding=encoding, @@ -62,28 +62,30 @@ def packb(o, default=None, encoding='utf-8', unicode_errors='struct', return packer.pack(o) def unpack(stream, object_hook=None, list_hook=None, use_list=True, - encoding=None, unicode_errors='strict', - object_pairs_hook=None): + encoding=None, unicode_errors='strict', + object_pairs_hook=None): """ Unpack an object from `stream`. Raises `ExtraData` when `stream` has extra bytes. """ unpacker = Unpacker(stream, object_hook=object_hook, list_hook=list_hook, - use_list=use_list, encoding=encoding, unicode_errors=unicode_errors, - object_pairs_hook=object_pairs_hook) + use_list=use_list, + encoding=encoding, unicode_errors=unicode_errors, + object_pairs_hook=object_pairs_hook) ret = unpacker._fb_unpack() if unpacker._fb_got_extradata(): raise ExtraData(ret, unpacker._fb_get_extradata()) return ret def unpackb(packed, object_hook=None, list_hook=None, use_list=True, - encoding=None, unicode_errors='strict', - object_pairs_hook=None): + encoding=None, unicode_errors='strict', + object_pairs_hook=None): """ Unpack an object from `packed`. Raises `ExtraData` when `packed` contains extra bytes. """ unpacker = Unpacker(None, object_hook=object_hook, list_hook=list_hook, - use_list=use_list, encoding=encoding, unicode_errors=unicode_errors, - object_pairs_hook=object_pairs_hook) + use_list=use_list, + encoding=encoding, unicode_errors=unicode_errors, + object_pairs_hook=object_pairs_hook) unpacker.feed(packed) ret = unpacker._fb_unpack() if unpacker._fb_got_extradata(): @@ -141,8 +143,8 @@ class Unpacker(object): """ def __init__(self, file_like=None, read_size=0, use_list=True, - object_hook=None, object_pairs_hook=None, list_hook=None, - encoding=None, unicode_errors='strict', max_buffer_size=0): + object_hook=None, object_pairs_hook=None, list_hook=None, + encoding=None, unicode_errors='strict', max_buffer_size=0): if file_like is None: self._fb_feeding = True else: @@ -174,8 +176,8 @@ class Unpacker(object): if object_pairs_hook is not None and not callable(object_pairs_hook): raise ValueError('`object_pairs_hook` is not callable') if object_hook is not None and object_pairs_hook is not None: - raise ValueError("object_pairs_hook and object_hook are mutually "+ - "exclusive") + raise ValueError("object_pairs_hook and object_hook are mutually " + "exclusive") def feed(self, next_bytes): if isinstance(next_bytes, array.array): @@ -394,7 +396,7 @@ class Unpacker(object): class Packer(object): def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', - use_single_float=False, autoreset=True): + use_single_float=False, autoreset=True): self.use_float = use_single_float self.autoreset = autoreset self.encoding = encoding @@ -404,6 +406,7 @@ class Packer(object): if not callable(default): raise TypeError("default must be callable") self._default = default + def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT): if nest_limit < 0: raise PackValueError("recursion limit exceeded") @@ -461,10 +464,11 @@ class Packer(object): return if isinstance(obj, dict): return self._fb_pack_map_pairs(len(obj), obj.iteritems(), - nest_limit - 1) + nest_limit - 1) if self._default is not None: return self._pack(self._default(obj), nest_limit - 1) raise TypeError("Cannot serialize %r" % obj) + def pack(self, obj): self._pack(obj) ret = self.buffer.getvalue() @@ -473,6 +477,7 @@ class Packer(object): elif USING_STRINGBUILDER: self.buffer = StringIO(ret) return ret + def pack_map_pairs(self, pairs): self._fb_pack_map_pairs(len(pairs), pairs) ret = self.buffer.getvalue() @@ -481,6 +486,7 @@ class Packer(object): elif USING_STRINGBUILDER: self.buffer = StringIO(ret) return ret + def pack_array_header(self, n): self._fb_pack_array_header(n) ret = self.buffer.getvalue() @@ -489,6 +495,7 @@ class Packer(object): elif USING_STRINGBUILDER: self.buffer = StringIO(ret) return ret + def pack_map_header(self, n): self._fb_pack_map_header(n) ret = self.buffer.getvalue() @@ -497,6 +504,7 @@ class Packer(object): elif USING_STRINGBUILDER: self.buffer = StringIO(ret) return ret + def _fb_pack_array_header(self, n): if n <= 0x0f: return self.buffer.write(chr(0x90 + n)) @@ -505,6 +513,7 @@ class Packer(object): if n <= 0xffffffff: return self.buffer.write(struct.pack(">BI", 0xdd, n)) raise PackValueError("Array is too large") + def _fb_pack_map_header(self, n): if n <= 0x0f: return self.buffer.write(chr(0x80 + n)) @@ -513,12 +522,15 @@ class Packer(object): if n <= 0xffffffff: return self.buffer.write(struct.pack(">BI", 0xdf, n)) raise PackValueError("Dict is too large") + def _fb_pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT): self._fb_pack_map_header(n) for (k, v) in pairs: self._pack(k, nest_limit - 1) self._pack(v, nest_limit - 1) + def bytes(self): return self.buffer.getvalue() + def reset(self): self.buffer = StringIO() From cbabeebc95e9e42c0356e089b742588a4de75d56 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 29 Jan 2013 14:47:16 +0900 Subject: [PATCH 0665/1172] Use MSGPACK_PUREPYTHON envvar to test fallback module --- .travis.yml | 2 +- msgpack/__init__.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2e6fc56..e536fdc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,4 +12,4 @@ install: - cython --cplus msgpack/_packer.pyx - cython --cplus msgpack/_unpacker.pyx -script: "tox" +script: "tox && MSGPACK_PUREPYTHON=x tox" diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 49a32d9..77f6b81 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -2,11 +2,15 @@ from msgpack._version import version from msgpack.exceptions import * -try: - from msgpack._packer import pack, packb, Packer - from msgpack._unpacker import unpack, unpackb, Unpacker -except ImportError: +import os +if os.environ.get('MSGPACK_PUREPYTHON'): from msgpack.fallback import pack, packb, Packer, unpack, unpackb, Unpacker +else: + try: + from msgpack._packer import pack, packb, Packer + from msgpack._unpacker import unpack, unpackb, Unpacker + except ImportError: + from msgpack.fallback import pack, packb, Packer, unpack, unpackb, Unpacker # alias for compatibility to simplejson/marshal/pickle. load = unpack From 8d6a387dff10dd2150aa86cd96e2bece26546268 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 29 Jan 2013 15:10:22 +0900 Subject: [PATCH 0666/1172] fallback: Support Python 3. --- msgpack/fallback.py | 54 ++++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 3d733f4..ac6dbf9 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -4,13 +4,28 @@ import sys import array import struct +if sys.version_info[0] == 3: + PY3 = True + int_types = int + Unicode = str + xrange = range + def dict_iteritems(d): + return d.items() +else: + PY3 = False + int_types = (int, long) + Unicode = unicode + def dict_iteritems(d): + return d.iteritems() + + if hasattr(sys, 'pypy_version_info'): # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own # StringBuilder is fastest. from __pypy__.builders import StringBuilder USING_STRINGBUILDER = True class StringIO(object): - def __init__(self, s=''): + def __init__(self, s=b''): if s: self.builder = StringBuilder(len(s)) self.builder.append(s) @@ -22,10 +37,7 @@ if hasattr(sys, 'pypy_version_info'): return self.builder.build() else: USING_STRINGBUILDER = False - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO + from io import BytesIO as StringIO from msgpack.exceptions import ( BufferFull, @@ -156,7 +168,7 @@ class Unpacker(object): self._fb_buf_o = 0 self._fb_buf_i = 0 self._fb_buf_n = 0 - self.max_buffer_size = (sys.maxint if max_buffer_size == 0 + self.max_buffer_size = (2**31-1 if max_buffer_size == 0 else max_buffer_size) self.read_size = (read_size if read_size != 0 else min(self.max_buffer_size, 2048)) @@ -221,7 +233,7 @@ class Unpacker(object): bufs = self._fb_buffers[self._fb_buf_i:] if bufs: bufs[0] = bufs[0][self._fb_buf_o:] - return ''.join(bufs) + return b''.join(bufs) def _fb_read(self, n, write_bytes=None): if (write_bytes is None and self._fb_buf_i < len(self._fb_buffers) @@ -229,7 +241,7 @@ class Unpacker(object): self._fb_buf_o += n return self._fb_buffers[self._fb_buf_i][ self._fb_buf_o-n:self._fb_buf_o] - ret = '' + ret = b'' while len(ret) != n: if self._fb_buf_i == len(self._fb_buffers): if self._fb_feeding: @@ -255,11 +267,12 @@ class Unpacker(object): def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None): typ = TYPE_IMMEDIATE - b = ord(self._fb_read(1, write_bytes)) + c = self._fb_read(1, write_bytes) + b = ord(c) if b & 0b10000000 == 0: obj = b elif b & 0b11100000 == 0b11100000: - obj = struct.unpack("b", chr(b))[0] + obj = struct.unpack("b", c)[0] elif b & 0b11100000 == 0b10100000: n = b & 0b00011111 obj = self._fb_read(n, write_bytes) @@ -374,6 +387,7 @@ class Unpacker(object): return ret except OutOfData: raise StopIteration + __next__ = next def skip(self, write_bytes=None): self._fb_unpack(EX_SKIP, write_bytes) @@ -411,12 +425,12 @@ class Packer(object): if nest_limit < 0: raise PackValueError("recursion limit exceeded") if obj is None: - return self.buffer.write(chr(0xc0)) + return self.buffer.write(b"\xc0") if isinstance(obj, bool): if obj: - return self.buffer.write(chr(0xc3)) - return self.buffer.write(chr(0xc2)) - if isinstance(obj, int) or isinstance(obj, long): + return self.buffer.write(b"\xc3") + return self.buffer.write(b"\xc2") + if isinstance(obj, int_types): if 0 <= obj < 0x80: return self.buffer.write(struct.pack("B", obj)) if -0x20 <= obj < 0: @@ -438,12 +452,12 @@ class Packer(object): if -0x8000000000000000 <= obj < -0x80000000: return self.buffer.write(struct.pack(">Bq", 0xd3, obj)) raise PackValueError("Integer value out of range") - if isinstance(obj, str) or isinstance(obj, unicode): - if isinstance(obj, unicode): + if isinstance(obj, (Unicode, bytes)): + if isinstance(obj, Unicode): obj = obj.encode(self.encoding, self.unicode_errors) n = len(obj) if n <= 0x1f: - self.buffer.write(chr(0xa0 + n)) + self.buffer.write(struct.pack('B', 0xa0 + n)) return self.buffer.write(obj) if n <= 0xffff: self.buffer.write(struct.pack(">BH", 0xda, n)) @@ -463,7 +477,7 @@ class Packer(object): self._pack(obj[i], nest_limit - 1) return if isinstance(obj, dict): - return self._fb_pack_map_pairs(len(obj), obj.iteritems(), + return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj), nest_limit - 1) if self._default is not None: return self._pack(self._default(obj), nest_limit - 1) @@ -507,7 +521,7 @@ class Packer(object): def _fb_pack_array_header(self, n): if n <= 0x0f: - return self.buffer.write(chr(0x90 + n)) + return self.buffer.write(struct.pack('B', 0x90 + n)) if n <= 0xffff: return self.buffer.write(struct.pack(">BH", 0xdc, n)) if n <= 0xffffffff: @@ -516,7 +530,7 @@ class Packer(object): def _fb_pack_map_header(self, n): if n <= 0x0f: - return self.buffer.write(chr(0x80 + n)) + return self.buffer.write(struct.pack('B', 0x80 + n)) if n <= 0xffff: return self.buffer.write(struct.pack(">BH", 0xde, n)) if n <= 0xffffffff: From 266eaf813d4e958dce5a2a8c4a84babf331369f0 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 29 Jan 2013 15:13:20 +0900 Subject: [PATCH 0667/1172] changelog: describe purepython fallback. --- ChangeLog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 55e296f..fdbb3ea 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -10,6 +10,7 @@ Inconpatible Changes Changes ------- +* Pure Python fallback module is added. (thanks to bwesterb) * Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman) * Add capturing feature. You can pass the writable object to ``Unpacker.unpack()`` as a second parameter. From 95dfec808a700c99144e454fbf9d98be420f2c51 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 3 Feb 2013 00:02:37 +0900 Subject: [PATCH 0668/1172] Add simple benchmark. --- benchmark/benchmark.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 benchmark/benchmark.py diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py new file mode 100644 index 0000000..1439289 --- /dev/null +++ b/benchmark/benchmark.py @@ -0,0 +1,32 @@ +from msgpack import fallback +try: + from msgpack import _unpacker, _packer + has_ext = True +except ImportError: + has_ext = False +import timeit + + +def profile(name, func): + times = timeit.repeat(func, number=1000, repeat=4) + times = ', '.join(["%8f" % t for t in times]) + print("%-30s %40s" % (name, times)) + + +def simple(name, data): + if has_ext: + profile("packing %s (ext)" % name, lambda: _packer.packb(data)) + profile('packing %s (fallback)' % name, lambda: fallback.packb(data)) + + data = fallback.packb(data) + if has_ext: + profile('unpacking %s (ext)' % name, lambda: _unpacker.unpackb(data)) + profile('unpacking %s (fallback)' % name, lambda: fallback.unpackb(data)) + +def main(): + simple("integers", [7]*10000) + simple("bytes", [b'x'*n for n in range(100)]*10) + simple("lists", [[]]*10000) + simple("dicts", [{}]*10000) + +main() From 0536d1bd0cb5c781b41e8cf7ede66448396dc993 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 3 Feb 2013 00:11:26 +0900 Subject: [PATCH 0669/1172] Don't compile extension module when running on pypy --- setup.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index d4808d6..888de09 100644 --- a/setup.py +++ b/setup.py @@ -75,18 +75,19 @@ else: macros = [('__LITTLE_ENDIAN__', '1')] ext_modules = [] -ext_modules.append(Extension('msgpack._packer', - sources=['msgpack/_packer.cpp'], - libraries=libraries, - include_dirs=['.'], - define_macros=macros, - )) -ext_modules.append(Extension('msgpack._unpacker', - sources=['msgpack/_unpacker.cpp'], - libraries=libraries, - include_dirs=['.'], - define_macros=macros, - )) +if not hasattr(sys, 'pypy_version_info'): + ext_modules.append(Extension('msgpack._packer', + sources=['msgpack/_packer.cpp'], + libraries=libraries, + include_dirs=['.'], + define_macros=macros, + )) + ext_modules.append(Extension('msgpack._unpacker', + sources=['msgpack/_unpacker.cpp'], + libraries=libraries, + include_dirs=['.'], + define_macros=macros, + )) del libraries, macros From 22920baae6957e2259a82d0595c2b97fc58fcd02 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 3 Feb 2013 00:20:00 +0900 Subject: [PATCH 0670/1172] Fix minor bugs and tuning unpacking dict. --- msgpack/fallback.py | 25 ++++++++++++++++--------- test/test_pack.py | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index ac6dbf9..c10c6ac 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -360,16 +360,19 @@ class Unpacker(object): self._fb_unpack(EX_SKIP, write_bytes) self._fb_unpack(EX_SKIP, write_bytes) return - ret = [] - for i in xrange(n): - ret.append((self._fb_unpack(EX_CONSTRUCT, write_bytes), - self._fb_unpack(EX_CONSTRUCT, write_bytes))) if self.object_pairs_hook is not None: - ret = self.object_pairs_hook(ret) + ret = self.object_pairs_hook( + (self._fb_unpack(EX_CONSTRUCT, write_bytes), + self._fb_unpack(EX_CONSTRUCT, write_bytes)) + for _ in xrange(n) + ) else: - ret = dict(ret) - if self.object_hook is not None: - ret = self.object_hook(ret) + ret = {} + for _ in xrange(n): + key = self._fb_unpack(EX_CONSTRUCT, write_bytes) + ret[key] = self._fb_unpack(EX_CONSTRUCT, write_bytes) + if self.object_hook is not None: + ret = self.object_hook(ret) return ret if execute == EX_SKIP: return @@ -421,7 +424,7 @@ class Packer(object): raise TypeError("default must be callable") self._default = default - def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT): + def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance): if nest_limit < 0: raise PackValueError("recursion limit exceeded") if obj is None: @@ -454,6 +457,10 @@ class Packer(object): raise PackValueError("Integer value out of range") if isinstance(obj, (Unicode, bytes)): if isinstance(obj, Unicode): + if self.encoding is None: + raise TypeError( + "Can't encode unicode string: " + "no encoding is specified") obj = obj.encode(self.encoding, self.unicode_errors) n = len(obj) if n <= 0x1f: diff --git a/test/test_pack.py b/test/test_pack.py index 8f4117c..3225f41 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -151,7 +151,7 @@ def test_odict(): od = odict(seq) assert unpackb(packb(od), use_list=1) == dict(seq) def pair_hook(seq): - return seq + return list(seq) assert unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1) == seq From 1951b197b547c3f12b755790717d799272fbeb34 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 3 Feb 2013 00:52:05 +0900 Subject: [PATCH 0671/1172] Skip compile error for extension modules. --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 888de09..1055a61 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,12 @@ class BuildExt(build_ext): print("Install Cython >= 0.16 or install msgpack from PyPI.") print("Falling back to pure Python implementation.") return - return build_ext.build_extension(self, ext) + try: + return build_ext.build_extension(self, ext) + except Exception as e: + print("WARNING: Failed to compile extensiom modules.") + print("msgpack uses fallback pure python implementation.") + print(e) exec(open('msgpack/_version.py').read()) From a865f8f7e94ea5b484771c1be3fe57ff3a63aa2a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 4 Feb 2013 15:14:30 +0900 Subject: [PATCH 0672/1172] Use _private names for non public data members. (fix #44) --- msgpack/fallback.py | 151 ++++++++++++++++++++++---------------------- 1 file changed, 76 insertions(+), 75 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index c10c6ac..a834229 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -168,18 +168,18 @@ class Unpacker(object): self._fb_buf_o = 0 self._fb_buf_i = 0 self._fb_buf_n = 0 - self.max_buffer_size = (2**31-1 if max_buffer_size == 0 - else max_buffer_size) - self.read_size = (read_size if read_size != 0 - else min(self.max_buffer_size, 2048)) - if read_size > self.max_buffer_size: + self._max_buffer_size = (2**31-1 if max_buffer_size == 0 + else max_buffer_size) + self._read_size = (read_size if read_size != 0 + else min(self._max_buffer_size, 2048)) + if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") - self.encoding = encoding - self.unicode_errors = unicode_errors - self.use_list = use_list - self.list_hook = list_hook - self.object_hook = object_hook - self.object_pairs_hook = object_pairs_hook + self._encoding = encoding + self._unicode_errors = unicode_errors + self._use_list = use_list + self._list_hook = list_hook + self._object_hook = object_hook + self._object_pairs_hook = object_pairs_hook if list_hook is not None and not callable(list_hook): raise ValueError('`list_hook` is not callable') @@ -195,7 +195,7 @@ class Unpacker(object): if isinstance(next_bytes, array.array): next_bytes = next_bytes.tostring() assert self._fb_feeding - if self._fb_buf_n + len(next_bytes) > self.max_buffer_size: + if self._fb_buf_n + len(next_bytes) > self._max_buffer_size: raise BufferFull self._fb_buf_n += len(next_bytes) self._fb_buffers.append(next_bytes) @@ -246,7 +246,7 @@ class Unpacker(object): if self._fb_buf_i == len(self._fb_buffers): if self._fb_feeding: break - tmp = self.file_like.read(self.read_size) + tmp = self.file_like.read(self._read_size) if not tmp: break self._fb_buffers.append(tmp) @@ -349,10 +349,10 @@ class Unpacker(object): ret = [] for i in xrange(n): ret.append(self._fb_unpack(EX_CONSTRUCT, write_bytes)) - if self.list_hook is not None: - ret = self.list_hook(ret) + if self._list_hook is not None: + ret = self._list_hook(ret) # TODO is the interaction between `list_hook` and `use_list` ok? - return ret if self.use_list else tuple(ret) + return ret if self._use_list else tuple(ret) if typ == TYPE_MAP: if execute == EX_SKIP: for i in xrange(n): @@ -360,8 +360,8 @@ class Unpacker(object): self._fb_unpack(EX_SKIP, write_bytes) self._fb_unpack(EX_SKIP, write_bytes) return - if self.object_pairs_hook is not None: - ret = self.object_pairs_hook( + if self._object_pairs_hook is not None: + ret = self._object_pairs_hook( (self._fb_unpack(EX_CONSTRUCT, write_bytes), self._fb_unpack(EX_CONSTRUCT, write_bytes)) for _ in xrange(n) @@ -371,14 +371,14 @@ class Unpacker(object): for _ in xrange(n): key = self._fb_unpack(EX_CONSTRUCT, write_bytes) ret[key] = self._fb_unpack(EX_CONSTRUCT, write_bytes) - if self.object_hook is not None: - ret = self.object_hook(ret) + if self._object_hook is not None: + ret = self._object_hook(ret) return ret if execute == EX_SKIP: return if typ == TYPE_RAW: - if self.encoding is not None: - obj = obj.decode(self.encoding, self.unicode_errors) + if self._encoding is not None: + obj = obj.decode(self._encoding, self._unicode_errors) return obj assert typ == TYPE_IMMEDIATE return obj @@ -411,14 +411,15 @@ class Unpacker(object): self._fb_consume() return ret + class Packer(object): def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, autoreset=True): - self.use_float = use_single_float - self.autoreset = autoreset - self.encoding = encoding - self.unicode_errors = unicode_errors - self.buffer = StringIO() + self._use_float = use_single_float + self._autoreset = autoreset + self._encoding = encoding + self._unicode_errors = unicode_errors + self._buffer = StringIO() if default is not None: if not callable(default): raise TypeError("default must be callable") @@ -428,55 +429,55 @@ class Packer(object): if nest_limit < 0: raise PackValueError("recursion limit exceeded") if obj is None: - return self.buffer.write(b"\xc0") + return self._buffer.write(b"\xc0") if isinstance(obj, bool): if obj: - return self.buffer.write(b"\xc3") - return self.buffer.write(b"\xc2") + return self._buffer.write(b"\xc3") + return self._buffer.write(b"\xc2") if isinstance(obj, int_types): if 0 <= obj < 0x80: - return self.buffer.write(struct.pack("B", obj)) + return self._buffer.write(struct.pack("B", obj)) if -0x20 <= obj < 0: - return self.buffer.write(struct.pack("b", obj)) + return self._buffer.write(struct.pack("b", obj)) if 0x80 <= obj <= 0xff: - return self.buffer.write(struct.pack("BB", 0xcc, obj)) + return self._buffer.write(struct.pack("BB", 0xcc, obj)) if -0x80 <= obj < 0: - return self.buffer.write(struct.pack(">Bb", 0xd0, obj)) + return self._buffer.write(struct.pack(">Bb", 0xd0, obj)) if 0xff < obj <= 0xffff: - return self.buffer.write(struct.pack(">BH", 0xcd, obj)) + return self._buffer.write(struct.pack(">BH", 0xcd, obj)) if -0x8000 <= obj < -0x80: - return self.buffer.write(struct.pack(">Bh", 0xd1, obj)) + return self._buffer.write(struct.pack(">Bh", 0xd1, obj)) if 0xffff < obj <= 0xffffffff: - return self.buffer.write(struct.pack(">BI", 0xce, obj)) + return self._buffer.write(struct.pack(">BI", 0xce, obj)) if -0x80000000 <= obj < -0x8000: - return self.buffer.write(struct.pack(">Bi", 0xd2, obj)) + return self._buffer.write(struct.pack(">Bi", 0xd2, obj)) if 0xffffffff < obj <= 0xffffffffffffffff: - return self.buffer.write(struct.pack(">BQ", 0xcf, obj)) + return self._buffer.write(struct.pack(">BQ", 0xcf, obj)) if -0x8000000000000000 <= obj < -0x80000000: - return self.buffer.write(struct.pack(">Bq", 0xd3, obj)) + return self._buffer.write(struct.pack(">Bq", 0xd3, obj)) raise PackValueError("Integer value out of range") if isinstance(obj, (Unicode, bytes)): if isinstance(obj, Unicode): - if self.encoding is None: + if self._encoding is None: raise TypeError( "Can't encode unicode string: " "no encoding is specified") - obj = obj.encode(self.encoding, self.unicode_errors) + obj = obj.encode(self._encoding, self._unicode_errors) n = len(obj) if n <= 0x1f: - self.buffer.write(struct.pack('B', 0xa0 + n)) - return self.buffer.write(obj) + self._buffer.write(struct.pack('B', 0xa0 + n)) + return self._buffer.write(obj) if n <= 0xffff: - self.buffer.write(struct.pack(">BH", 0xda, n)) - return self.buffer.write(obj) + self._buffer.write(struct.pack(">BH", 0xda, n)) + return self._buffer.write(obj) if n <= 0xffffffff: - self.buffer.write(struct.pack(">BI", 0xdb, n)) - return self.buffer.write(obj) + self._buffer.write(struct.pack(">BI", 0xdb, n)) + return self._buffer.write(obj) raise PackValueError("String is too large") if isinstance(obj, float): - if self.use_float: - return self.buffer.write(struct.pack(">Bf", 0xca, obj)) - return self.buffer.write(struct.pack(">Bd", 0xcb, obj)) + if self._use_float: + return self._buffer.write(struct.pack(">Bf", 0xca, obj)) + return self._buffer.write(struct.pack(">Bd", 0xcb, obj)) if isinstance(obj, list) or isinstance(obj, tuple): n = len(obj) self._fb_pack_array_header(n) @@ -492,56 +493,56 @@ class Packer(object): def pack(self, obj): self._pack(obj) - ret = self.buffer.getvalue() - if self.autoreset: - self.buffer = StringIO() + ret = self._buffer.getvalue() + if self._autoreset: + self._buffer = StringIO() elif USING_STRINGBUILDER: - self.buffer = StringIO(ret) + self._buffer = StringIO(ret) return ret def pack_map_pairs(self, pairs): self._fb_pack_map_pairs(len(pairs), pairs) - ret = self.buffer.getvalue() - if self.autoreset: - self.buffer = StringIO() + ret = self._buffer.getvalue() + if self._autoreset: + self._buffer = StringIO() elif USING_STRINGBUILDER: - self.buffer = StringIO(ret) + self._buffer = StringIO(ret) return ret def pack_array_header(self, n): self._fb_pack_array_header(n) - ret = self.buffer.getvalue() - if self.autoreset: - self.buffer = StringIO() + ret = self._buffer.getvalue() + if self._autoreset: + self._buffer = StringIO() elif USING_STRINGBUILDER: - self.buffer = StringIO(ret) + self._buffer = StringIO(ret) return ret def pack_map_header(self, n): self._fb_pack_map_header(n) - ret = self.buffer.getvalue() - if self.autoreset: - self.buffer = StringIO() + ret = self._buffer.getvalue() + if self._autoreset: + self._buffer = StringIO() elif USING_STRINGBUILDER: - self.buffer = StringIO(ret) + self._buffer = StringIO(ret) return ret def _fb_pack_array_header(self, n): if n <= 0x0f: - return self.buffer.write(struct.pack('B', 0x90 + n)) + return self._buffer.write(struct.pack('B', 0x90 + n)) if n <= 0xffff: - return self.buffer.write(struct.pack(">BH", 0xdc, n)) + return self._buffer.write(struct.pack(">BH", 0xdc, n)) if n <= 0xffffffff: - return self.buffer.write(struct.pack(">BI", 0xdd, n)) + return self._buffer.write(struct.pack(">BI", 0xdd, n)) raise PackValueError("Array is too large") def _fb_pack_map_header(self, n): if n <= 0x0f: - return self.buffer.write(struct.pack('B', 0x80 + n)) + return self._buffer.write(struct.pack('B', 0x80 + n)) if n <= 0xffff: - return self.buffer.write(struct.pack(">BH", 0xde, n)) + return self._buffer.write(struct.pack(">BH", 0xde, n)) if n <= 0xffffffff: - return self.buffer.write(struct.pack(">BI", 0xdf, n)) + return self._buffer.write(struct.pack(">BI", 0xdf, n)) raise PackValueError("Dict is too large") def _fb_pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT): @@ -551,7 +552,7 @@ class Packer(object): self._pack(v, nest_limit - 1) def bytes(self): - return self.buffer.getvalue() + return self._buffer.getvalue() def reset(self): - self.buffer = StringIO() + self._buffer = StringIO() From df6449f17366b9d608895766cc1a151167460ae0 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 16 Feb 2013 09:28:29 +0900 Subject: [PATCH 0673/1172] 0.3.0 --- msgpack/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index e750485..5999ede 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 3, 0, 'dev1') +version = (0, 3, 0) From 626ae51017cdbf62a2c7b00b338747d1b2070513 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 16 Feb 2013 14:03:39 +0900 Subject: [PATCH 0674/1172] README: s/nosetest/pytest/ --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 270b8ff..f2e5962 100644 --- a/README.rst +++ b/README.rst @@ -192,10 +192,10 @@ Another way to unpacking such object is using ``object_pairs_hook``. TEST ---- -MessagePack uses `nosetest` for testing. +MessagePack uses `pytest` for testing. Run test with following command: - $ nosetests test + $ py.test .. From 3f12846d40e035573993cc9c5195bf3607ee98bb Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sat, 16 Feb 2013 12:08:14 -0800 Subject: [PATCH 0675/1172] On PyPy, preallocate lists When deserealizing arrays, preallocate the resulting list at the correct size. --- msgpack/fallback.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index a834229..0016bd1 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -22,6 +22,7 @@ else: if hasattr(sys, 'pypy_version_info'): # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own # StringBuilder is fastest. + from __pypy__ import newlist_hint from __pypy__.builders import StringBuilder USING_STRINGBUILDER = True class StringIO(object): @@ -38,6 +39,7 @@ if hasattr(sys, 'pypy_version_info'): else: USING_STRINGBUILDER = False from io import BytesIO as StringIO + newlist_hint = lambda size: [] from msgpack.exceptions import ( BufferFull, @@ -346,7 +348,7 @@ class Unpacker(object): # TODO check whether we need to call `list_hook` self._fb_unpack(EX_SKIP, write_bytes) return - ret = [] + ret = newlist_hint(n) for i in xrange(n): ret.append(self._fb_unpack(EX_CONSTRUCT, write_bytes)) if self._list_hook is not None: From 952403319478ac4a95c4d5d9c85908ba6dda5672 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 21 Feb 2013 14:01:12 +0900 Subject: [PATCH 0676/1172] skip reserved byte. --- msgpack/fallback.py | 14 ++++++- msgpack/unpack_template.h | 85 ++++++++++++++++++++------------------- 2 files changed, 56 insertions(+), 43 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 0016bd1..311f16b 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -57,6 +57,7 @@ TYPE_IMMEDIATE = 0 TYPE_ARRAY = 1 TYPE_MAP = 2 TYPE_RAW = 3 +TYPE_RESERVED = 4 DEFAULT_RECURSE_LIMIT=511 @@ -267,8 +268,10 @@ class Unpacker(object): write_bytes(ret) return ret - def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None): + def _read_header(self, execute=EX_CONSTRUCT, write_bytes=None): typ = TYPE_IMMEDIATE + n = 0 + obj = None c = self._fb_read(1, write_bytes) b = ord(c) if b & 0b10000000 == 0: @@ -332,7 +335,14 @@ class Unpacker(object): n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] typ = TYPE_MAP else: - raise UnpackValueError("Unknown header: 0x%x" % b) + typ = TYPE_RESERVED + return typ, n, obj + + def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None): + typ = TYPE_RESERVED + while typ == TYPE_RESERVED: + typ, n, obj = self._read_header(self, execute, write_bytes) + if execute == EX_READ_ARRAY_HEADER: if typ != TYPE_ARRAY: raise UnpackValueError("Expected array") diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 83b6918..8fd179d 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -422,53 +422,56 @@ _end: template msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) { - assert(len >= *off); - uint32_t size; - const unsigned char *const p = (unsigned char*)data + *off; + for (;;) { + if (len < *off) { + return 0; + } + uint32_t size; + const unsigned char *const p = (unsigned char*)data + *off; #define inc_offset(inc) \ - if (len - *off < inc) \ - return 0; \ - *off += inc; + if (len - *off < inc) \ + return 0; \ + *off += inc; - switch (*p) { - case var_offset: - inc_offset(3); - size = _msgpack_load16(uint16_t, p + 1); - break; - case var_offset + 1: - inc_offset(5); - size = _msgpack_load32(uint32_t, p + 1); - break; + switch (*p) { + case var_offset: + inc_offset(3); + size = _msgpack_load16(uint16_t, p + 1); + break; + case var_offset + 1: + inc_offset(5); + size = _msgpack_load32(uint32_t, p + 1); + break; #ifdef USE_CASE_RANGE - case fixed_offset + 0x0 ... fixed_offset + 0xf: + case fixed_offset + 0x0 ... fixed_offset + 0xf: #else - case fixed_offset + 0x0: - case fixed_offset + 0x1: - case fixed_offset + 0x2: - case fixed_offset + 0x3: - case fixed_offset + 0x4: - case fixed_offset + 0x5: - case fixed_offset + 0x6: - case fixed_offset + 0x7: - case fixed_offset + 0x8: - case fixed_offset + 0x9: - case fixed_offset + 0xa: - case fixed_offset + 0xb: - case fixed_offset + 0xc: - case fixed_offset + 0xd: - case fixed_offset + 0xe: - case fixed_offset + 0xf: + case fixed_offset + 0x0: + case fixed_offset + 0x1: + case fixed_offset + 0x2: + case fixed_offset + 0x3: + case fixed_offset + 0x4: + case fixed_offset + 0x5: + case fixed_offset + 0x6: + case fixed_offset + 0x7: + case fixed_offset + 0x8: + case fixed_offset + 0x9: + case fixed_offset + 0xa: + case fixed_offset + 0xb: + case fixed_offset + 0xc: + case fixed_offset + 0xd: + case fixed_offset + 0xe: + case fixed_offset + 0xf: #endif - ++*off; - size = ((unsigned int)*p) & 0x0f; - break; - default: - PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); - return -1; - } - msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj); - return 1; + ++*off; + size = ((unsigned int)*p) & 0x0f; + break; + default: + continue; + } + msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj); + return 1; + } } #undef SWITCH_RANGE_BEGIN From ff3342aeedb9d21897b0b7275dd68d781f149995 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 21 Feb 2013 16:02:33 +0900 Subject: [PATCH 0677/1172] Skip reserved byte --- msgpack/unpack_template.h | 298 +++++++++++++++++++++----------------- test/test_reserved.py | 8 + 2 files changed, 171 insertions(+), 135 deletions(-) create mode 100644 test/test_reserved.py diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 8fd179d..6f86695 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -60,10 +60,10 @@ msgpack_unpack_struct_decl(_context) { unsigned int trail; unsigned int top; /* - msgpack_unpack_struct(_stack)* stack; - unsigned int stack_size; - msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE]; - */ + msgpack_unpack_struct(_stack)* stack; + unsigned int stack_size; + msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE]; + */ msgpack_unpack_struct(_stack) stack[MSGPACK_EMBED_STACK_SIZE]; }; @@ -74,20 +74,20 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) ctx->trail = 0; ctx->top = 0; /* - ctx->stack = ctx->embed_stack; - ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; - */ + ctx->stack = ctx->embed_stack; + ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; + */ ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); } /* -msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx) -{ - if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { - free(ctx->stack); - } -} -*/ + msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx) + { + if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { + free(ctx->stack); + } + } + */ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) { @@ -109,8 +109,8 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c unsigned int top = ctx->top; msgpack_unpack_struct(_stack)* stack = ctx->stack; /* - unsigned int stack_size = ctx->stack_size; - */ + unsigned int stack_size = ctx->stack_size; + */ msgpack_unpack_user* user = &ctx->user; msgpack_unpack_object obj; @@ -119,7 +119,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c int ret; #define construct_cb(name) \ - construct && msgpack_unpack_callback(name) + construct && msgpack_unpack_callback(name) #define push_simple_value(func) \ if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \ @@ -129,7 +129,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c goto _push #define push_variable_value(func, base, pos, len) \ if(construct_cb(func)(user, \ - (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ + (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ goto _push #define again_fixed_trail(_cs, trail_len) \ @@ -155,24 +155,24 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ /*printf("stack push %d\n", top);*/ \ /* FIXME \ - if(top >= stack_size) { \ - if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \ - size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \ - size_t nsize = csize * 2; \ - msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \ - if(tmp == NULL) { goto _failed; } \ - memcpy(tmp, ctx->stack, csize); \ - ctx->stack = stack = tmp; \ - ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ - } else { \ - size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \ - msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \ - if(tmp == NULL) { goto _failed; } \ - ctx->stack = stack = tmp; \ - ctx->stack_size = stack_size = stack_size * 2; \ - } \ - } \ - */ \ + if(top >= stack_size) { \ + if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \ + size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \ + size_t nsize = csize * 2; \ + msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \ + if(tmp == NULL) { goto _failed; } \ + memcpy(tmp, ctx->stack, csize); \ + ctx->stack = stack = tmp; \ + ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ + } else { \ + size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \ + msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \ + if(tmp == NULL) { goto _failed; } \ + ctx->stack = stack = tmp; \ + ctx->stack_size = stack_size = stack_size * 2; \ + } \ + } \ + */ \ goto _header_again #define NEXT_CS(p) \ @@ -195,7 +195,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c switch(cs) { case CS_HEADER: SWITCH_RANGE_BEGIN - SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum + SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum push_fixed_value(_uint8, *(uint8_t*)p); SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum push_fixed_value(_int8, *(int8_t*)p); @@ -203,18 +203,24 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c switch(*p) { case 0xc0: // nil push_simple_value(_nil); - //case 0xc1: // string - // again_terminal_trail(NEXT_CS(p), p+1); + // reserved. + case 0xc1: // string + fprintf(stderr, "skipping %x\n", (int)*p); + goto _header_again; + // again_terminal_trail(NEXT_CS(p), p+1); case 0xc2: // false push_simple_value(_false); case 0xc3: // true push_simple_value(_true); - //case 0xc4: - //case 0xc5: - //case 0xc6: - //case 0xc7: - //case 0xc8: - //case 0xc9: + // reserved. + case 0xc4: + case 0xc5: + case 0xc6: + case 0xc7: + case 0xc8: + case 0xc9: + fprintf(stderr, "skipping %x\n", (int)*p); + goto _header_again; case 0xca: // float case 0xcb: // double case 0xcc: // unsigned int 8 @@ -226,12 +232,15 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c case 0xd2: // signed int 32 case 0xd3: // signed int 64 again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); - //case 0xd4: - //case 0xd5: - //case 0xd6: // big integer 16 - //case 0xd7: // big integer 32 - //case 0xd8: // big float 16 - //case 0xd9: // big float 32 + // reserved. + case 0xd4: + case 0xd5: + case 0xd6: // big integer 16 + case 0xd7: // big integer 32 + case 0xd8: // big float 16 + case 0xd9: // big float 32 + fprintf(stderr, "skipping %x\n", (int)*p); + goto _header_again; case 0xda: // raw 16 case 0xdb: // raw 32 case 0xdc: // array 16 @@ -240,6 +249,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c case 0xdf: // map 32 again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); default: + fprintf(stderr, "failed %x\n", (int)*p); goto _failed; } SWITCH_RANGE(0xa0, 0xbf) // FixRaw @@ -250,128 +260,129 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); SWITCH_RANGE_DEFAULT - goto _failed; + fprintf(stderr, "failed2 %x\n", (int)*p); + goto _failed; SWITCH_RANGE_END - // end CS_HEADER + // end CS_HEADER - _fixed_trail_again: - ++p; + _fixed_trail_again: + ++p; default: if((size_t)(pe - p) < trail) { goto _out; } n = p; p += trail - 1; switch(cs) { - //case CS_ - //case CS_ + //case CS_ + //case CS_ case CS_FLOAT: { - union { uint32_t i; float f; } mem; - mem.i = _msgpack_load32(uint32_t,n); - push_fixed_value(_float, mem.f); } + union { uint32_t i; float f; } mem; + mem.i = _msgpack_load32(uint32_t,n); + push_fixed_value(_float, mem.f); } case CS_DOUBLE: { - union { uint64_t i; double f; } mem; - mem.i = _msgpack_load64(uint64_t,n); + union { uint64_t i; double f; } mem; + mem.i = _msgpack_load64(uint64_t,n); #if defined(__arm__) && !(__ARM_EABI__) // arm-oabi - // https://github.com/msgpack/msgpack-perl/pull/1 - mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL); + // https://github.com/msgpack/msgpack-perl/pull/1 + mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL); #endif - push_fixed_value(_double, mem.f); } + push_fixed_value(_double, mem.f); } case CS_UINT_8: - push_fixed_value(_uint8, *(uint8_t*)n); + push_fixed_value(_uint8, *(uint8_t*)n); case CS_UINT_16: - push_fixed_value(_uint16, _msgpack_load16(uint16_t,n)); + push_fixed_value(_uint16, _msgpack_load16(uint16_t,n)); case CS_UINT_32: - push_fixed_value(_uint32, _msgpack_load32(uint32_t,n)); + push_fixed_value(_uint32, _msgpack_load32(uint32_t,n)); case CS_UINT_64: - push_fixed_value(_uint64, _msgpack_load64(uint64_t,n)); + push_fixed_value(_uint64, _msgpack_load64(uint64_t,n)); case CS_INT_8: - push_fixed_value(_int8, *(int8_t*)n); + push_fixed_value(_int8, *(int8_t*)n); case CS_INT_16: - push_fixed_value(_int16, _msgpack_load16(int16_t,n)); + push_fixed_value(_int16, _msgpack_load16(int16_t,n)); case CS_INT_32: - push_fixed_value(_int32, _msgpack_load32(int32_t,n)); + push_fixed_value(_int32, _msgpack_load32(int32_t,n)); case CS_INT_64: - push_fixed_value(_int64, _msgpack_load64(int64_t,n)); + push_fixed_value(_int64, _msgpack_load64(int64_t,n)); - //case CS_ - //case CS_ - //case CS_BIG_INT_16: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero); - //case CS_BIG_INT_32: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero); - //case ACS_BIG_INT_VALUE: - //_big_int_zero: - // // FIXME - // push_variable_value(_big_int, data, n, trail); + //case CS_ + //case CS_ + //case CS_BIG_INT_16: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero); + //case CS_BIG_INT_32: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero); + //case ACS_BIG_INT_VALUE: + //_big_int_zero: + // // FIXME + // push_variable_value(_big_int, data, n, trail); - //case CS_BIG_FLOAT_16: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero); - //case CS_BIG_FLOAT_32: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero); - //case ACS_BIG_FLOAT_VALUE: - //_big_float_zero: - // // FIXME - // push_variable_value(_big_float, data, n, trail); + //case CS_BIG_FLOAT_16: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero); + //case CS_BIG_FLOAT_32: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero); + //case ACS_BIG_FLOAT_VALUE: + //_big_float_zero: + // // FIXME + // push_variable_value(_big_float, data, n, trail); case CS_RAW_16: - again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero); + again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero); case CS_RAW_32: - again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero); + again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero); case ACS_RAW_VALUE: - _raw_zero: - push_variable_value(_raw, data, n, trail); +_raw_zero: + push_variable_value(_raw, data, n, trail); case CS_ARRAY_16: - start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM); + start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM); case CS_ARRAY_32: - /* FIXME security guard */ - start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM); + /* FIXME security guard */ + start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM); case CS_MAP_16: - start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY); + start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY); case CS_MAP_32: - /* FIXME security guard */ - start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); + /* FIXME security guard */ + start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); default: - goto _failed; + goto _failed; } } _push: - if(top == 0) { goto _finish; } - c = &stack[top-1]; - switch(c->ct) { - case CT_ARRAY_ITEM: - if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } - if(++c->count == c->size) { - obj = c->obj; - if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; } - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - goto _header_again; - case CT_MAP_KEY: - c->map_key = obj; - c->ct = CT_MAP_VALUE; - goto _header_again; - case CT_MAP_VALUE: - if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } - if(++c->count == c->size) { - obj = c->obj; - if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; } - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - c->ct = CT_MAP_KEY; - goto _header_again; + if(top == 0) { goto _finish; } + c = &stack[top-1]; + switch(c->ct) { + case CT_ARRAY_ITEM: + if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } + if(++c->count == c->size) { + obj = c->obj; + if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; } + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + goto _header_again; + case CT_MAP_KEY: + c->map_key = obj; + c->ct = CT_MAP_VALUE; + goto _header_again; + case CT_MAP_VALUE: + if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } + if(++c->count == c->size) { + obj = c->obj; + if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; } + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + c->ct = CT_MAP_KEY; + goto _header_again; - default: - goto _failed; - } + default: + goto _failed; + } _header_again: cs = CS_HEADER; @@ -431,7 +442,7 @@ msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx #define inc_offset(inc) \ if (len - *off < inc) \ - return 0; \ + return 0; \ *off += inc; switch (*p) { @@ -466,8 +477,25 @@ msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx ++*off; size = ((unsigned int)*p) & 0x0f; break; - default: + // reserved: + case 0xc1: + case 0xc4: + case 0xc5: + case 0xc6: + case 0xc7: + case 0xc8: + case 0xc9: + case 0xd4: + case 0xd5: + case 0xd6: // big integer 16 + case 0xd7: // big integer 32 + case 0xd8: // big float 16 + case 0xd9: // big float 32 + ++*off; continue; + default: + PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); + return -1; } msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj); return 1; diff --git a/test/test_reserved.py b/test/test_reserved.py new file mode 100644 index 0000000..56b4455 --- /dev/null +++ b/test/test_reserved.py @@ -0,0 +1,8 @@ +import msgpack + +reserved_bytes = b"\xc1\xc4\xc5\xc6\xc7\xc8\xc9\xd4\xd5\xd6\xd7\xd8\xd9" + +def test_skip_reserved(): + packed_list = msgpack.packb([]) + for b in reserved_bytes: + assert msgpack.unpackb(b+packed_list, use_list=1) == [] From 43dd224d52eb4711c68101b40b910d252dacc290 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 21 Feb 2013 16:04:58 +0900 Subject: [PATCH 0678/1172] Fix test for Python 3. --- test/test_reserved.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/test_reserved.py b/test/test_reserved.py index 56b4455..5094064 100644 --- a/test/test_reserved.py +++ b/test/test_reserved.py @@ -1,6 +1,20 @@ import msgpack -reserved_bytes = b"\xc1\xc4\xc5\xc6\xc7\xc8\xc9\xd4\xd5\xd6\xd7\xd8\xd9" +reserved_bytes = [ + b"\xc1", + b"\xc4", + b"\xc5", + b"\xc6", + b"\xc7", + b"\xc8", + b"\xc9", + b"\xd4", + b"\xd5", + b"\xd6", + b"\xd7", + b"\xd8", + b"\xd9", + ] def test_skip_reserved(): packed_list = msgpack.packb([]) From d766820421c9c67972f899cf6333fbb00e595162 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 21 Feb 2013 16:55:42 +0900 Subject: [PATCH 0679/1172] Fix easy bug. --- msgpack/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 311f16b..8dae9a7 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -341,7 +341,7 @@ class Unpacker(object): def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None): typ = TYPE_RESERVED while typ == TYPE_RESERVED: - typ, n, obj = self._read_header(self, execute, write_bytes) + typ, n, obj = self._read_header(execute, write_bytes) if execute == EX_READ_ARRAY_HEADER: if typ != TYPE_ARRAY: From f0fd90a759f07bccfd36e4f9eac8c4b545fd47a4 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 22 Feb 2013 17:41:52 +0900 Subject: [PATCH 0680/1172] Fix exception incompatibility. --- msgpack/_unpacker.pyx | 4 +--- msgpack/fallback.py | 5 ++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index b83a6e4..aeda02a 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -113,10 +113,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, if off < buf_len: raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) return obj - elif ret < 0: - raise ValueError("Unpack failed: error = %d" % (ret,)) else: - raise UnpackValueError + raise UnpackValueError("Unpack failed: error = %d" % (ret,)) def unpack(object stream, object object_hook=None, object list_hook=None, diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 8dae9a7..f893e1a 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -102,7 +102,10 @@ def unpackb(packed, object_hook=None, list_hook=None, use_list=True, encoding=encoding, unicode_errors=unicode_errors, object_pairs_hook=object_pairs_hook) unpacker.feed(packed) - ret = unpacker._fb_unpack() + try: + ret = unpacker._fb_unpack() + except OutOfData: + raise UnpackValueError("Data is not enough.") if unpacker._fb_got_extradata(): raise ExtraData(ret, unpacker._fb_get_extradata()) return ret From 5c51203d14ba2420d98f7c9f85a21b3a7b89e1b9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 23 Feb 2013 18:00:54 +0900 Subject: [PATCH 0681/1172] Revert "Fix test for Python 3." This reverts commit 43dd224d52eb4711c68101b40b910d252dacc290. --- test/test_reserved.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/test/test_reserved.py b/test/test_reserved.py index 5094064..56b4455 100644 --- a/test/test_reserved.py +++ b/test/test_reserved.py @@ -1,20 +1,6 @@ import msgpack -reserved_bytes = [ - b"\xc1", - b"\xc4", - b"\xc5", - b"\xc6", - b"\xc7", - b"\xc8", - b"\xc9", - b"\xd4", - b"\xd5", - b"\xd6", - b"\xd7", - b"\xd8", - b"\xd9", - ] +reserved_bytes = b"\xc1\xc4\xc5\xc6\xc7\xc8\xc9\xd4\xd5\xd6\xd7\xd8\xd9" def test_skip_reserved(): packed_list = msgpack.packb([]) From a6859791a2048f5b2aae1ece03a603af63d69633 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 23 Feb 2013 18:01:43 +0900 Subject: [PATCH 0682/1172] Revert "Skip reserved byte" This reverts commit ff3342aeedb9d21897b0b7275dd68d781f149995. --- msgpack/unpack_template.h | 298 +++++++++++++++++--------------------- test/test_reserved.py | 8 - 2 files changed, 135 insertions(+), 171 deletions(-) delete mode 100644 test/test_reserved.py diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 6f86695..8fd179d 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -60,10 +60,10 @@ msgpack_unpack_struct_decl(_context) { unsigned int trail; unsigned int top; /* - msgpack_unpack_struct(_stack)* stack; - unsigned int stack_size; - msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE]; - */ + msgpack_unpack_struct(_stack)* stack; + unsigned int stack_size; + msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE]; + */ msgpack_unpack_struct(_stack) stack[MSGPACK_EMBED_STACK_SIZE]; }; @@ -74,20 +74,20 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) ctx->trail = 0; ctx->top = 0; /* - ctx->stack = ctx->embed_stack; - ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; - */ + ctx->stack = ctx->embed_stack; + ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; + */ ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); } /* - msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx) - { - if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { - free(ctx->stack); - } - } - */ +msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx) +{ + if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { + free(ctx->stack); + } +} +*/ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) { @@ -109,8 +109,8 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c unsigned int top = ctx->top; msgpack_unpack_struct(_stack)* stack = ctx->stack; /* - unsigned int stack_size = ctx->stack_size; - */ + unsigned int stack_size = ctx->stack_size; + */ msgpack_unpack_user* user = &ctx->user; msgpack_unpack_object obj; @@ -119,7 +119,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c int ret; #define construct_cb(name) \ - construct && msgpack_unpack_callback(name) + construct && msgpack_unpack_callback(name) #define push_simple_value(func) \ if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \ @@ -129,7 +129,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c goto _push #define push_variable_value(func, base, pos, len) \ if(construct_cb(func)(user, \ - (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ + (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ goto _push #define again_fixed_trail(_cs, trail_len) \ @@ -155,24 +155,24 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ /*printf("stack push %d\n", top);*/ \ /* FIXME \ - if(top >= stack_size) { \ - if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \ - size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \ - size_t nsize = csize * 2; \ - msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \ - if(tmp == NULL) { goto _failed; } \ - memcpy(tmp, ctx->stack, csize); \ - ctx->stack = stack = tmp; \ - ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ - } else { \ - size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \ - msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \ - if(tmp == NULL) { goto _failed; } \ - ctx->stack = stack = tmp; \ - ctx->stack_size = stack_size = stack_size * 2; \ - } \ - } \ - */ \ + if(top >= stack_size) { \ + if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \ + size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \ + size_t nsize = csize * 2; \ + msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \ + if(tmp == NULL) { goto _failed; } \ + memcpy(tmp, ctx->stack, csize); \ + ctx->stack = stack = tmp; \ + ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ + } else { \ + size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \ + msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \ + if(tmp == NULL) { goto _failed; } \ + ctx->stack = stack = tmp; \ + ctx->stack_size = stack_size = stack_size * 2; \ + } \ + } \ + */ \ goto _header_again #define NEXT_CS(p) \ @@ -195,7 +195,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c switch(cs) { case CS_HEADER: SWITCH_RANGE_BEGIN - SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum + SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum push_fixed_value(_uint8, *(uint8_t*)p); SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum push_fixed_value(_int8, *(int8_t*)p); @@ -203,24 +203,18 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c switch(*p) { case 0xc0: // nil push_simple_value(_nil); - // reserved. - case 0xc1: // string - fprintf(stderr, "skipping %x\n", (int)*p); - goto _header_again; - // again_terminal_trail(NEXT_CS(p), p+1); + //case 0xc1: // string + // again_terminal_trail(NEXT_CS(p), p+1); case 0xc2: // false push_simple_value(_false); case 0xc3: // true push_simple_value(_true); - // reserved. - case 0xc4: - case 0xc5: - case 0xc6: - case 0xc7: - case 0xc8: - case 0xc9: - fprintf(stderr, "skipping %x\n", (int)*p); - goto _header_again; + //case 0xc4: + //case 0xc5: + //case 0xc6: + //case 0xc7: + //case 0xc8: + //case 0xc9: case 0xca: // float case 0xcb: // double case 0xcc: // unsigned int 8 @@ -232,15 +226,12 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c case 0xd2: // signed int 32 case 0xd3: // signed int 64 again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); - // reserved. - case 0xd4: - case 0xd5: - case 0xd6: // big integer 16 - case 0xd7: // big integer 32 - case 0xd8: // big float 16 - case 0xd9: // big float 32 - fprintf(stderr, "skipping %x\n", (int)*p); - goto _header_again; + //case 0xd4: + //case 0xd5: + //case 0xd6: // big integer 16 + //case 0xd7: // big integer 32 + //case 0xd8: // big float 16 + //case 0xd9: // big float 32 case 0xda: // raw 16 case 0xdb: // raw 32 case 0xdc: // array 16 @@ -249,7 +240,6 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c case 0xdf: // map 32 again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); default: - fprintf(stderr, "failed %x\n", (int)*p); goto _failed; } SWITCH_RANGE(0xa0, 0xbf) // FixRaw @@ -260,129 +250,128 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); SWITCH_RANGE_DEFAULT - fprintf(stderr, "failed2 %x\n", (int)*p); - goto _failed; + goto _failed; SWITCH_RANGE_END - // end CS_HEADER + // end CS_HEADER - _fixed_trail_again: - ++p; + _fixed_trail_again: + ++p; default: if((size_t)(pe - p) < trail) { goto _out; } n = p; p += trail - 1; switch(cs) { - //case CS_ - //case CS_ + //case CS_ + //case CS_ case CS_FLOAT: { - union { uint32_t i; float f; } mem; - mem.i = _msgpack_load32(uint32_t,n); - push_fixed_value(_float, mem.f); } + union { uint32_t i; float f; } mem; + mem.i = _msgpack_load32(uint32_t,n); + push_fixed_value(_float, mem.f); } case CS_DOUBLE: { - union { uint64_t i; double f; } mem; - mem.i = _msgpack_load64(uint64_t,n); + union { uint64_t i; double f; } mem; + mem.i = _msgpack_load64(uint64_t,n); #if defined(__arm__) && !(__ARM_EABI__) // arm-oabi - // https://github.com/msgpack/msgpack-perl/pull/1 - mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL); + // https://github.com/msgpack/msgpack-perl/pull/1 + mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL); #endif - push_fixed_value(_double, mem.f); } + push_fixed_value(_double, mem.f); } case CS_UINT_8: - push_fixed_value(_uint8, *(uint8_t*)n); + push_fixed_value(_uint8, *(uint8_t*)n); case CS_UINT_16: - push_fixed_value(_uint16, _msgpack_load16(uint16_t,n)); + push_fixed_value(_uint16, _msgpack_load16(uint16_t,n)); case CS_UINT_32: - push_fixed_value(_uint32, _msgpack_load32(uint32_t,n)); + push_fixed_value(_uint32, _msgpack_load32(uint32_t,n)); case CS_UINT_64: - push_fixed_value(_uint64, _msgpack_load64(uint64_t,n)); + push_fixed_value(_uint64, _msgpack_load64(uint64_t,n)); case CS_INT_8: - push_fixed_value(_int8, *(int8_t*)n); + push_fixed_value(_int8, *(int8_t*)n); case CS_INT_16: - push_fixed_value(_int16, _msgpack_load16(int16_t,n)); + push_fixed_value(_int16, _msgpack_load16(int16_t,n)); case CS_INT_32: - push_fixed_value(_int32, _msgpack_load32(int32_t,n)); + push_fixed_value(_int32, _msgpack_load32(int32_t,n)); case CS_INT_64: - push_fixed_value(_int64, _msgpack_load64(int64_t,n)); + push_fixed_value(_int64, _msgpack_load64(int64_t,n)); - //case CS_ - //case CS_ - //case CS_BIG_INT_16: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero); - //case CS_BIG_INT_32: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero); - //case ACS_BIG_INT_VALUE: - //_big_int_zero: - // // FIXME - // push_variable_value(_big_int, data, n, trail); + //case CS_ + //case CS_ + //case CS_BIG_INT_16: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero); + //case CS_BIG_INT_32: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero); + //case ACS_BIG_INT_VALUE: + //_big_int_zero: + // // FIXME + // push_variable_value(_big_int, data, n, trail); - //case CS_BIG_FLOAT_16: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero); - //case CS_BIG_FLOAT_32: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero); - //case ACS_BIG_FLOAT_VALUE: - //_big_float_zero: - // // FIXME - // push_variable_value(_big_float, data, n, trail); + //case CS_BIG_FLOAT_16: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero); + //case CS_BIG_FLOAT_32: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero); + //case ACS_BIG_FLOAT_VALUE: + //_big_float_zero: + // // FIXME + // push_variable_value(_big_float, data, n, trail); case CS_RAW_16: - again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero); + again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero); case CS_RAW_32: - again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero); + again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero); case ACS_RAW_VALUE: -_raw_zero: - push_variable_value(_raw, data, n, trail); + _raw_zero: + push_variable_value(_raw, data, n, trail); case CS_ARRAY_16: - start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM); + start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM); case CS_ARRAY_32: - /* FIXME security guard */ - start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM); + /* FIXME security guard */ + start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM); case CS_MAP_16: - start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY); + start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY); case CS_MAP_32: - /* FIXME security guard */ - start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); + /* FIXME security guard */ + start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); default: - goto _failed; + goto _failed; } } _push: - if(top == 0) { goto _finish; } - c = &stack[top-1]; - switch(c->ct) { - case CT_ARRAY_ITEM: - if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } - if(++c->count == c->size) { - obj = c->obj; - if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; } - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - goto _header_again; - case CT_MAP_KEY: - c->map_key = obj; - c->ct = CT_MAP_VALUE; - goto _header_again; - case CT_MAP_VALUE: - if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } - if(++c->count == c->size) { - obj = c->obj; - if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; } - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - c->ct = CT_MAP_KEY; - goto _header_again; - - default: - goto _failed; + if(top == 0) { goto _finish; } + c = &stack[top-1]; + switch(c->ct) { + case CT_ARRAY_ITEM: + if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } + if(++c->count == c->size) { + obj = c->obj; + if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; } + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; } + goto _header_again; + case CT_MAP_KEY: + c->map_key = obj; + c->ct = CT_MAP_VALUE; + goto _header_again; + case CT_MAP_VALUE: + if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } + if(++c->count == c->size) { + obj = c->obj; + if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; } + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + c->ct = CT_MAP_KEY; + goto _header_again; + + default: + goto _failed; + } _header_again: cs = CS_HEADER; @@ -442,7 +431,7 @@ msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx #define inc_offset(inc) \ if (len - *off < inc) \ - return 0; \ + return 0; \ *off += inc; switch (*p) { @@ -477,25 +466,8 @@ msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx ++*off; size = ((unsigned int)*p) & 0x0f; break; - // reserved: - case 0xc1: - case 0xc4: - case 0xc5: - case 0xc6: - case 0xc7: - case 0xc8: - case 0xc9: - case 0xd4: - case 0xd5: - case 0xd6: // big integer 16 - case 0xd7: // big integer 32 - case 0xd8: // big float 16 - case 0xd9: // big float 32 - ++*off; - continue; default: - PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); - return -1; + continue; } msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj); return 1; diff --git a/test/test_reserved.py b/test/test_reserved.py deleted file mode 100644 index 56b4455..0000000 --- a/test/test_reserved.py +++ /dev/null @@ -1,8 +0,0 @@ -import msgpack - -reserved_bytes = b"\xc1\xc4\xc5\xc6\xc7\xc8\xc9\xd4\xd5\xd6\xd7\xd8\xd9" - -def test_skip_reserved(): - packed_list = msgpack.packb([]) - for b in reserved_bytes: - assert msgpack.unpackb(b+packed_list, use_list=1) == [] From 38a9ad98c9e8e8b73efb9726554737a60ec215c9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 23 Feb 2013 18:11:46 +0900 Subject: [PATCH 0683/1172] Revert skipping reserved byte. --- msgpack/fallback.py | 9 ++--- msgpack/unpack_template.h | 85 +++++++++++++++++++-------------------- 2 files changed, 44 insertions(+), 50 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index f893e1a..e9dddd6 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -48,7 +48,7 @@ from msgpack.exceptions import ( PackValueError, ExtraData) -EX_SKIP = 0 +EX_SKIP = 0 EX_CONSTRUCT = 1 EX_READ_ARRAY_HEADER = 2 EX_READ_MAP_HEADER = 3 @@ -57,7 +57,6 @@ TYPE_IMMEDIATE = 0 TYPE_ARRAY = 1 TYPE_MAP = 2 TYPE_RAW = 3 -TYPE_RESERVED = 4 DEFAULT_RECURSE_LIMIT=511 @@ -338,13 +337,11 @@ class Unpacker(object): n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] typ = TYPE_MAP else: - typ = TYPE_RESERVED + raise UnpackValueError("Unknown header: 0x%x" % b) return typ, n, obj def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None): - typ = TYPE_RESERVED - while typ == TYPE_RESERVED: - typ, n, obj = self._read_header(execute, write_bytes) + typ, n, obj = self._read_header(execute, write_bytes) if execute == EX_READ_ARRAY_HEADER: if typ != TYPE_ARRAY: diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 8fd179d..83b6918 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -422,56 +422,53 @@ _end: template msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) { - for (;;) { - if (len < *off) { - return 0; - } - uint32_t size; - const unsigned char *const p = (unsigned char*)data + *off; + assert(len >= *off); + uint32_t size; + const unsigned char *const p = (unsigned char*)data + *off; #define inc_offset(inc) \ - if (len - *off < inc) \ - return 0; \ - *off += inc; + if (len - *off < inc) \ + return 0; \ + *off += inc; - switch (*p) { - case var_offset: - inc_offset(3); - size = _msgpack_load16(uint16_t, p + 1); - break; - case var_offset + 1: - inc_offset(5); - size = _msgpack_load32(uint32_t, p + 1); - break; + switch (*p) { + case var_offset: + inc_offset(3); + size = _msgpack_load16(uint16_t, p + 1); + break; + case var_offset + 1: + inc_offset(5); + size = _msgpack_load32(uint32_t, p + 1); + break; #ifdef USE_CASE_RANGE - case fixed_offset + 0x0 ... fixed_offset + 0xf: + case fixed_offset + 0x0 ... fixed_offset + 0xf: #else - case fixed_offset + 0x0: - case fixed_offset + 0x1: - case fixed_offset + 0x2: - case fixed_offset + 0x3: - case fixed_offset + 0x4: - case fixed_offset + 0x5: - case fixed_offset + 0x6: - case fixed_offset + 0x7: - case fixed_offset + 0x8: - case fixed_offset + 0x9: - case fixed_offset + 0xa: - case fixed_offset + 0xb: - case fixed_offset + 0xc: - case fixed_offset + 0xd: - case fixed_offset + 0xe: - case fixed_offset + 0xf: + case fixed_offset + 0x0: + case fixed_offset + 0x1: + case fixed_offset + 0x2: + case fixed_offset + 0x3: + case fixed_offset + 0x4: + case fixed_offset + 0x5: + case fixed_offset + 0x6: + case fixed_offset + 0x7: + case fixed_offset + 0x8: + case fixed_offset + 0x9: + case fixed_offset + 0xa: + case fixed_offset + 0xb: + case fixed_offset + 0xc: + case fixed_offset + 0xd: + case fixed_offset + 0xe: + case fixed_offset + 0xf: #endif - ++*off; - size = ((unsigned int)*p) & 0x0f; - break; - default: - continue; - } - msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj); - return 1; - } + ++*off; + size = ((unsigned int)*p) & 0x0f; + break; + default: + PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); + return -1; + } + msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj); + return 1; } #undef SWITCH_RANGE_BEGIN From f4cb6fb87749491bce5e3e6b2d7bfafdf7764435 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 24 Feb 2013 18:06:36 +0900 Subject: [PATCH 0684/1172] Add apidoc --- docs/Makefile | 156 +++++++++++++++++++++++++++ docs/api.rst | 38 +++++++ docs/conf.py | 285 +++++++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 11 ++ 4 files changed, 490 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/api.rst create mode 100644 docs/conf.py create mode 100644 docs/index.rst diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..10d4d4b --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,156 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/msgpack.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/msgpack.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/msgpack" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/msgpack" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +serve: html + cd _build/html && python3.3 -m http.server diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000..a26e7c7 --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,38 @@ +API reference +=============== + +.. module:: msgpack + +.. autofunction:: pack + +:func:`dump` is alias for :func:`pack` + +.. autofunction:: packb + +:func:`dumps` is alias for :func:`packb` + +.. autofunction:: unpack + +:func:`load` is alias for :func:`unpack` + +.. autofunction:: unpackb + +:func:`loads` is alias for :func:`unpackb` + +.. autoclass:: Packer + :members: + +.. autoclass:: Unpacker + :members: + +exceptions +----------- + +These exceptions are accessible via `msgpack` package. +(For example, `msgpack.OutOfData` is shortcut for `msgpack.exceptions.OutOfData`) + +.. automodule:: msgpack.exceptions + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..0821ad4 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,285 @@ +# -*- coding: utf-8 -*- +# +# msgpack documentation build configuration file, created by +# sphinx-quickstart on Sun Feb 24 14:20:50 2013. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'msgpack' +copyright = u'2013, Author' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '' +# The full version, including alpha/beta/rc tags. +release = '' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinxdoc' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'msgpackdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'msgpack.tex', u'msgpack Documentation', + u'Author', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'msgpack', u'msgpack Documentation', + [u'Author'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'msgpack', u'msgpack Documentation', + u'Author', 'msgpack', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + + +# -- Options for Epub output --------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = u'msgpack' +epub_author = u'Author' +epub_publisher = u'Author' +epub_copyright = u'2013, Author' + +# The language of the text. It defaults to the language option +# or en if the language is not set. +#epub_language = '' + +# The scheme of the identifier. Typical schemes are ISBN or URL. +#epub_scheme = '' + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +#epub_identifier = '' + +# A unique identification for the text. +#epub_uid = '' + +# A tuple containing the cover image and cover page html template filenames. +#epub_cover = () + +# HTML files that should be inserted before the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_pre_files = [] + +# HTML files shat should be inserted after the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_post_files = [] + +# A list of files that should not be packed into the epub file. +#epub_exclude_files = [] + +# The depth of the table of contents in toc.ncx. +#epub_tocdepth = 3 + +# Allow duplicate toc entries. +#epub_tocdup = True diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..5dcff6f --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,11 @@ +msgpack document +================== + +`MessagePack `_ is a efficient format for inter +language data exchange. + +.. toctree:: + :maxdepth: 1 + + ../README + api From 7991530cecf061a50792bcb423153c6a150bc9e6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 24 Feb 2013 18:06:50 +0900 Subject: [PATCH 0685/1172] Update README --- README.rst | 77 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/README.rst b/README.rst index f2e5962..da2f023 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ -=========================== -MessagePack Python Binding -=========================== +======================= +MessagePack for Python +======================= :author: INADA Naoki :version: 0.3.0 @@ -9,19 +9,42 @@ MessagePack Python Binding .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png :target: https://travis-ci.org/#!/msgpack/msgpack-python -WHAT IT IS ----------- +What's this +------------ `MessagePack `_ is a fast, compact binary serialization format, suitable for similar data to JSON. This package provides CPython bindings for reading and writing MessagePack data. -NOTE for msgpack 0.2.x users +Install +--------- +You can use ``pip`` or ``easy_install`` to install msgpack:: + + $ easy_install msgpack-python + or + $ pip install msgpack-python + +PyPy +^^^^^ + +msgpack-python provides pure python implementation. +PyPy can use this. + +Windows +^^^^^^^ + +When you can't use binary distribution, you need to install Visual Studio +or Windows SDK on Windows. (NOTE: Visual C++ Express 2010 doesn't support +amd64. Windows SDK is recommanded way to build amd64 msgpack without any fee.) + +Without extension, using pure python implementation on CPython runs slowly. + +Note for msgpack 0.2.x users ---------------------------- The msgpack 0.3 have some incompatible changes. -The default value of ``use_list`` keyword argument is ``True`` from 0.3.x. +The default value of ``use_list`` keyword argument is ``True`` from 0.3. You should pass the argument explicitly for backward compatibility. `Unpacker.unpack()` and some unpack methods now raises `OutOfData` @@ -29,10 +52,10 @@ instead of `StopIteration`. `StopIteration` is used for iterator protocol only. -HOW TO USE +How to use ----------- -one-shot pack & unpack +One-shot pack & unpack ^^^^^^^^^^^^^^^^^^^^^^ Use ``packb`` for packing and ``unpackb`` for unpacking. @@ -60,7 +83,7 @@ You should always pass the ``use_list`` keyword argument. See performance issues Read the docstring for other options. -streaming unpacking +Streaming unpacking ^^^^^^^^^^^^^^^^^^^ ``Unpacker`` is a "streaming unpacker". It unpacks multiple objects from one @@ -82,7 +105,7 @@ stream (or from bytes provided through its ``feed`` method). print unpacked -packing/unpacking of custom data type +Packing/unpacking of custom data type ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is also possible to pack/unpack custom data types. Here is an example for @@ -118,7 +141,7 @@ It is also possible to pack/unpack custom data types. Here is an example for key-value pairs. -advanced unpacking control +Advanced unpacking control ^^^^^^^^^^^^^^^^^^^^^^^^^^ As an alternative to iteration, ``Unpacker`` objects provide ``unpack``, @@ -150,27 +173,8 @@ callback function: unpacker.skip(bytestream.write) worker.send(bytestream.getvalue()) -INSTALL ---------- -You can use ``pip`` or ``easy_install`` to install msgpack:: - - $ easy_install msgpack-python - or - $ pip install msgpack-python - - -Windows -^^^^^^^ -msgpack provides some binary distribution for Windows. -You can install msgpack without compiler with them. - -When you can't use binary distribution, you need to install Visual Studio -or Windows SDK on Windows. (NOTE: Visual C++ Express 2010 doesn't support -amd64. Windows SDK is recommanded way to build amd64 msgpack without any fee.) - - -PERFORMANCE NOTE ------------------ +Note about performance +------------------------ GC ^^ @@ -179,8 +183,8 @@ CPython's GC starts when growing allocated object. This means unpacking may cause useless GC. You can use ``gc.disable()`` when unpacking large message. -use_list -^^^^^^^^^ +`use_list` option +^^^^^^^^^^^^^^^^^^ List is the default sequence type of Python. But tuple is lighter than list. You can use ``use_list=False`` while unpacking when performance is important. @@ -190,13 +194,12 @@ Python's dict can't use list as key and MessagePack allows array for key of mapp Another way to unpacking such object is using ``object_pairs_hook``. -TEST +Test ---- MessagePack uses `pytest` for testing. Run test with following command: $ py.test - .. vim: filetype=rst From 230537cf287e5b6985f1fd06ebb01a45776065e6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 25 Feb 2013 14:29:49 +0900 Subject: [PATCH 0686/1172] Update docs. --- docs/api.rst | 2 +- docs/conf.py | 6 +++--- docs/index.rst | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index a26e7c7..7efc04a 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1,5 +1,5 @@ API reference -=============== +============= .. module:: msgpack diff --git a/docs/conf.py b/docs/conf.py index 0821ad4..fba09b7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,16 +41,15 @@ master_doc = 'index' # General information about the project. project = u'msgpack' -copyright = u'2013, Author' +copyright = u'2013, INADA Naoki' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '' # The full version, including alpha/beta/rc tags. -release = '' +version = release = '0.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -61,6 +60,7 @@ release = '' #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' +today_fmt = "%Y-%m-%d" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/docs/index.rst b/docs/index.rst index 5dcff6f..72d4499 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,5 +7,4 @@ language data exchange. .. toctree:: :maxdepth: 1 - ../README api From 1e38bfa1235896f2df5d34211166276b48afa4b9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 25 Feb 2013 18:23:42 +0900 Subject: [PATCH 0687/1172] fallback: refactor --- msgpack/fallback.py | 58 ++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index e9dddd6..28ce036 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -60,46 +60,44 @@ TYPE_RAW = 3 DEFAULT_RECURSE_LIMIT=511 -def pack(o, stream, default=None, encoding='utf-8', unicode_errors='strict'): - """ Pack object `o` and write it to `stream` """ - packer = Packer(default=default, encoding=encoding, - unicode_errors=unicode_errors) +def pack(o, stream, **kwargs): + """ + Pack object `o` and write it to `stream` + + See :class:`Packer` for options. + """ + packer = Packer(**kwargs) stream.write(packer.pack(o)) -def packb(o, default=None, encoding='utf-8', unicode_errors='struct', - use_single_float=False): - """ Pack object `o` and return packed bytes """ - packer = Packer(default=default, - encoding=encoding, - unicode_errors=unicode_errors, - use_single_float=use_single_float) - return packer.pack(o) +def packb(o, **kwargs): + """ + Pack object `o` and return packed bytes -def unpack(stream, object_hook=None, list_hook=None, use_list=True, - encoding=None, unicode_errors='strict', - object_pairs_hook=None): - """ Unpack an object from `stream`. + See :class:`Packer` for options. + """ + return Packer(**kwargs).pack(o) - Raises `ExtraData` when `stream` has extra bytes. """ - unpacker = Unpacker(stream, object_hook=object_hook, list_hook=list_hook, - use_list=use_list, - encoding=encoding, unicode_errors=unicode_errors, - object_pairs_hook=object_pairs_hook) +def unpack(stream, **kwargs): + """ + Unpack an object from `stream`. + + Raises `ExtraData` when `packed` contains extra bytes. + See :class:`Unpacker` for options. + """ + unpacker = Unpacker(stream, **kwargs) ret = unpacker._fb_unpack() if unpacker._fb_got_extradata(): raise ExtraData(ret, unpacker._fb_get_extradata()) return ret -def unpackb(packed, object_hook=None, list_hook=None, use_list=True, - encoding=None, unicode_errors='strict', - object_pairs_hook=None): - """ Unpack an object from `packed`. +def unpackb(packed, **kwargs): + """ + Unpack an object from `packed`. - Raises `ExtraData` when `packed` contains extra bytes. """ - unpacker = Unpacker(None, object_hook=object_hook, list_hook=list_hook, - use_list=use_list, - encoding=encoding, unicode_errors=unicode_errors, - object_pairs_hook=object_pairs_hook) + Raises `ExtraData` when `packed` contains extra bytes. + See :class:`Unpacker` for options. + """ + unpacker = Unpacker(None, **kwargs) unpacker.feed(packed) try: ret = unpacker._fb_unpack() From 3ce005cf377eaeb5a517ce82698e4ae4381e67b9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 26 Feb 2013 09:20:44 +0900 Subject: [PATCH 0688/1172] better packer docstring --- Makefile | 12 ++++++++---- docs/Makefile | 2 +- docs/api.rst | 5 +++++ msgpack/_packer.pyx | 42 ++++++++++++++++++++++++++++-------------- msgpack/fallback.py | 24 ++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 875a3f6..da17b81 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,16 @@ .PHONY: test all python3 -all: +all: cython python setup.py build_ext -i -f - python setup.py build sdist -python3: +doc-serve: all + cd docs && make serve + +cython: + cython msgpack/*.pyx + +python3: cython python3 setup.py build_ext -i -f - python3 setup.py build sdist test: py.test test diff --git a/docs/Makefile b/docs/Makefile index 10d4d4b..427a980 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -10,7 +10,7 @@ BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +ALLSPHINXOPTS = -E -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . diff --git a/docs/api.rst b/docs/api.rst index 7efc04a..ccc7952 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -36,3 +36,8 @@ These exceptions are accessible via `msgpack` package. :undoc-members: :show-inheritance: +.. automodule:: msgpack.fallback + :members: + :undoc-members: + :show-inheritance: + diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index a5bc570..562c92c 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -39,9 +39,10 @@ cdef int DEFAULT_RECURSE_LIMIT=511 cdef class Packer(object): - """MessagePack Packer + """ + MessagePack Packer - usage: + usage:: packer = Packer() astream.write(packer.pack(a)) @@ -49,13 +50,18 @@ cdef class Packer(object): Packer's constructor has some keyword arguments: - * *defaut* - Convert user type to builtin type that Packer supports. - See also simplejson's document. - * *encoding* - Convert unicode to bytes with this encoding. (default: 'utf-8') - * *unicode_erros* - Error handler for encoding unicode. (default: 'strict') - * *use_single_float* - Use single precision float type for float. (default: False) - * *autoreset* - Reset buffer after each pack and return it's content as `bytes`. (default: True). - If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. + :param callable default: + Convert user type to builtin type that Packer supports. + See also simplejson's document. + :param str encoding: + Convert unicode to bytes with this encoding. (default: 'utf-8') + :param str unicode_erros: + Error handler for encoding unicode. (default: 'strict') + :param bool use_single_float: + Use single precision float type for float. (default: False) + :param bool autoreset: + Reset buffer after each pack and return it's content as `bytes`. (default: True). + If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. """ cdef msgpack_packer pk cdef object _default @@ -75,6 +81,8 @@ cdef class Packer(object): self.pk.length = 0 def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, bint autoreset=1): + """ + """ self.use_float = use_single_float self.autoreset = autoreset if default is not None: @@ -218,7 +226,7 @@ cdef class Packer(object): Pack *pairs* as msgpack map type. *pairs* should sequence of pair. - (`len(pairs)` and `for k, v in *pairs*:` should be supported.) + (`len(pairs)` and `for k, v in pairs:` should be supported.) """ cdef int ret = msgpack_pack_map(&self.pk, len(pairs)) if ret == 0: @@ -245,15 +253,21 @@ cdef class Packer(object): return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) -def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'): +def pack(object o, object stream, default=None, str encoding='utf-8', str unicode_errors='strict'): + """ + pack an object `o` and write it to stream) + + See :class:`Packer` for options. """ - pack an object `o` and write it to stream).""" packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors) stream.write(packer.pack(o)) -def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False): +def packb(object o, default=None, encoding='utf-8', str unicode_errors='strict', bint use_single_float=False): + """ + pack o and return packed bytes + + See :class:`Packer` for options. """ - pack o and return packed bytes.""" packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors, use_single_float=use_single_float) return packer.pack(o) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 28ce036..f9a2f5e 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -423,6 +423,30 @@ class Unpacker(object): class Packer(object): + """ + MessagePack Packer + + usage: + + packer = Packer() + astream.write(packer.pack(a)) + astream.write(packer.pack(b)) + + Packer's constructor has some keyword arguments: + + :param callable default: + Convert user type to builtin type that Packer supports. + See also simplejson's document. + :param str encoding: + Convert unicode to bytes with this encoding. (default: 'utf-8') + :param str unicode_erros: + Error handler for encoding unicode. (default: 'strict') + :param bool use_single_float: + Use single precision float type for float. (default: False) + :param bool autoreset: + Reset buffer after each pack and return it's content as `bytes`. (default: True). + If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. + """ def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, autoreset=True): self._use_float = use_single_float From 8e13598a36cbf56992454a04e1c4e92e4cfb15b8 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 26 Feb 2013 09:49:25 +0900 Subject: [PATCH 0689/1172] docs: better unpacker docstring. --- Makefile | 3 ++ docs/Makefile | 3 ++ msgpack/_unpacker.pyx | 64 +++++++++++++++++++++++++++---------------- 3 files changed, 46 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index da17b81..3fe278e 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,9 @@ all: cython doc-serve: all cd docs && make serve +doc: + cd docs && make zip + cython: cython msgpack/*.pyx diff --git a/docs/Makefile b/docs/Makefile index 427a980..0869604 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -154,3 +154,6 @@ doctest: serve: html cd _build/html && python3.3 -m http.server + +zip: html + cd _build/html && zip -r ../../../msgpack-doc.zip . diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index aeda02a..bd838f2 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -81,9 +81,12 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=1, encoding=None, unicode_errors="strict", object_pairs_hook=None, ): - """Unpack packed_bytes to object. Returns an unpacked object. + """ + Unpack packed_bytes to object. Returns an unpacked object. Raises `ValueError` when `packed` contains extra bytes. + + See :class:`Unpacker` for options. """ cdef template_context ctx cdef size_t off = 0 @@ -121,9 +124,12 @@ def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=1, encoding=None, unicode_errors="strict", object_pairs_hook=None, ): - """Unpack an object from `stream`. + """ + Unpack an object from `stream`. Raises `ValueError` when `stream` has extra bytes. + + See :class:`Unpacker` for options. """ return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, @@ -135,48 +141,58 @@ cdef class Unpacker(object): """ Streaming unpacker. - `file_like` is a file-like object having `.read(n)` method. - When `Unpacker` initialized with `file_like`, unpacker reads serialized data - from it and `.feed()` method is not usable. + arguments: - `read_size` is used as `file_like.read(read_size)`. - (default: min(1024**2, max_buffer_size)) + :param file_like: + File-like object having `.read(n)` method. + If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. - If `use_list` is true (default), msgpack list is deserialized to Python list. - Otherwise, it is deserialized to Python tuple. + :param int read_size: + Used as `file_like.read(read_size)`. (default: `min(1024**2, max_buffer_size)`) - `object_hook` is same to simplejson. If it is not None, it should be callable - and Unpacker calls it with a dict argument after deserializing a map. + :param bool use_list: + If true, unpack msgpack array to Python list. + Otherwise, unpack to Python tuple. (default: True) - `object_pairs_hook` is same to simplejson. If it is not None, it should be callable - and Unpacker calls it with a list of key-value pairs after deserializing a map. + :param callable object_hook: + When specified, it should be callable. + Unpacker calls it with a dict argument after unpacking msgpack map. + (See also simplejson) - `encoding` is encoding used for decoding msgpack bytes. If it is None (default), - msgpack bytes is deserialized to Python bytes. + :param callable object_pairs_hook: + When specified, it should be callable. + Unpacker calls it with a list of key-value pairs after unpacking msgpack map. + (See also simplejson) - `unicode_errors` is used for decoding bytes. + :param str encoding: + Encoding used for decoding msgpack raw. + If it is None (default), msgpack raw is deserialized to Python bytes. - `max_buffer_size` limits size of data waiting unpacked. - 0 means system's INT_MAX (default). - Raises `BufferFull` exception when it is insufficient. - You shoud set this parameter when unpacking data from untrasted source. + :param str unicode_errors: + Used for decoding msgpack raw with *encoding*. + (default: `'strict'`) + + :param int max_buffer_size: + Limits size of data waiting unpacked. 0 means system's INT_MAX (default). + Raises `BufferFull` exception when it is insufficient. + You shoud set this parameter when unpacking data from untrasted source. example of streaming deserialize from file-like object:: unpacker = Unpacker(file_like) for o in unpacker: - do_something(o) + process(o) example of streaming deserialize from socket:: unpacker = Unpacker() - while 1: + while True: buf = sock.recv(1024**2) if not buf: break unpacker.feed(buf) for o in unpacker: - do_something(o) + process(o) """ cdef template_context ctx cdef char* buf @@ -197,7 +213,7 @@ cdef class Unpacker(object): def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1, object object_hook=None, object object_pairs_hook=None, object list_hook=None, - encoding=None, unicode_errors='strict', int max_buffer_size=0, + str encoding=None, str unicode_errors='strict', int max_buffer_size=0, ): cdef char *cenc=NULL, *cerr=NULL From 5176e92d995f95b1b7db2554f6d44112ed04e8f8 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 26 Feb 2013 09:55:13 +0900 Subject: [PATCH 0690/1172] Fix typeerror. --- msgpack/_unpacker.pyx | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index bd838f2..daeb6d7 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -239,15 +239,17 @@ cdef class Unpacker(object): if encoding is not None: if isinstance(encoding, unicode): - encoding = encoding.encode('ascii') - self.encoding = encoding - cenc = PyBytes_AsString(encoding) + self.encoding = encoding.encode('ascii') + else: + self.encoding = encoding + cenc = PyBytes_AsString(self.encoding) if unicode_errors is not None: if isinstance(unicode_errors, unicode): - unicode_errors = unicode_errors.encode('ascii') - self.unicode_errors = unicode_errors - cerr = PyBytes_AsString(unicode_errors) + self.unicode_errors = unicode_errors.encode('ascii') + else: + self.unicode_errors = unicode_errors + cerr = PyBytes_AsString(self.unicode_errors) init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) From d2feb1362946c87609e3753a058ddbd0958ee0bd Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 26 Feb 2013 14:20:44 +0900 Subject: [PATCH 0691/1172] doc: remove fallback from api reference. --- docs/api.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index ccc7952..50a84c4 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -35,9 +35,3 @@ These exceptions are accessible via `msgpack` package. :members: :undoc-members: :show-inheritance: - -.. automodule:: msgpack.fallback - :members: - :undoc-members: - :show-inheritance: - From 58d8effc357ae06b68dec9f434c8039a6ddad0d3 Mon Sep 17 00:00:00 2001 From: jnothman Date: Wed, 27 Feb 2013 18:26:57 +1100 Subject: [PATCH 0692/1172] Copy note on OutOfData from pypi to ChangeLog --- ChangeLog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index fdbb3ea..fe64ff8 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -7,6 +7,8 @@ Inconpatible Changes * Default value of ``use_list`` is ``True`` for now. (It was ``False`` for 0.2.x) You should pass it explicitly for compatibility to 0.2.x. +* `Unpacker.unpack()` and some unpack methods now raise `OutOfData` instead of + `StopIteration`. `StopIteration` is used for iterator protocol only. Changes ------- From c91131f49f47b2af0349c1807d1c0f183d2b99b3 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 27 Feb 2013 20:37:07 +0900 Subject: [PATCH 0693/1172] remove msgpack_pack* macros --- msgpack/pack.h | 8 --- msgpack/pack_template.h | 120 +++++++++------------------------------- 2 files changed, 25 insertions(+), 103 deletions(-) diff --git a/msgpack/pack.h b/msgpack/pack.h index bb939d9..59764d9 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -90,14 +90,6 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ return 0; } -#define msgpack_pack_inline_func(name) \ - static inline int msgpack_pack ## name - -#define msgpack_pack_inline_func_cint(name) \ - static inline int msgpack_pack ## name - -#define msgpack_pack_user msgpack_packer* - #define msgpack_pack_append_buffer(user, buf, len) \ return msgpack_pack_write(user, (const char*)buf, len) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 65c959d..d6bf820 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -28,14 +28,6 @@ #define TAKE8_64(d) ((uint8_t*)&d)[7] #endif -#ifndef msgpack_pack_inline_func -#error msgpack_pack_inline_func template is not defined -#endif - -#ifndef msgpack_pack_user -#error msgpack_pack_user type is not defined -#endif - #ifndef msgpack_pack_append_buffer #error msgpack_pack_append_buffer callback is not defined #endif @@ -272,102 +264,42 @@ do { \ } while(0) -#ifdef msgpack_pack_inline_func_fixint - -msgpack_pack_inline_func_fixint(_uint8)(msgpack_pack_user x, uint8_t d) -{ - unsigned char buf[2] = {0xcc, TAKE8_8(d)}; - msgpack_pack_append_buffer(x, buf, 2); -} - -msgpack_pack_inline_func_fixint(_uint16)(msgpack_pack_user x, uint16_t d) -{ - unsigned char buf[3]; - buf[0] = 0xcd; _msgpack_store16(&buf[1], d); - msgpack_pack_append_buffer(x, buf, 3); -} - -msgpack_pack_inline_func_fixint(_uint32)(msgpack_pack_user x, uint32_t d) -{ - unsigned char buf[5]; - buf[0] = 0xce; _msgpack_store32(&buf[1], d); - msgpack_pack_append_buffer(x, buf, 5); -} - -msgpack_pack_inline_func_fixint(_uint64)(msgpack_pack_user x, uint64_t d) -{ - unsigned char buf[9]; - buf[0] = 0xcf; _msgpack_store64(&buf[1], d); - msgpack_pack_append_buffer(x, buf, 9); -} - -msgpack_pack_inline_func_fixint(_int8)(msgpack_pack_user x, int8_t d) -{ - unsigned char buf[2] = {0xd0, TAKE8_8(d)}; - msgpack_pack_append_buffer(x, buf, 2); -} - -msgpack_pack_inline_func_fixint(_int16)(msgpack_pack_user x, int16_t d) -{ - unsigned char buf[3]; - buf[0] = 0xd1; _msgpack_store16(&buf[1], d); - msgpack_pack_append_buffer(x, buf, 3); -} - -msgpack_pack_inline_func_fixint(_int32)(msgpack_pack_user x, int32_t d) -{ - unsigned char buf[5]; - buf[0] = 0xd2; _msgpack_store32(&buf[1], d); - msgpack_pack_append_buffer(x, buf, 5); -} - -msgpack_pack_inline_func_fixint(_int64)(msgpack_pack_user x, int64_t d) -{ - unsigned char buf[9]; - buf[0] = 0xd3; _msgpack_store64(&buf[1], d); - msgpack_pack_append_buffer(x, buf, 9); -} - -#undef msgpack_pack_inline_func_fixint -#endif - - -msgpack_pack_inline_func(_uint8)(msgpack_pack_user x, uint8_t d) +static inline int msgpack_pack_uint8(msgpack_packer* x, uint8_t d) { msgpack_pack_real_uint8(x, d); } -msgpack_pack_inline_func(_uint16)(msgpack_pack_user x, uint16_t d) +static inline int msgpack_pack_uint16(msgpack_packer* x, uint16_t d) { msgpack_pack_real_uint16(x, d); } -msgpack_pack_inline_func(_uint32)(msgpack_pack_user x, uint32_t d) +static inline int msgpack_pack_uint32(msgpack_packer* x, uint32_t d) { msgpack_pack_real_uint32(x, d); } -msgpack_pack_inline_func(_uint64)(msgpack_pack_user x, uint64_t d) +static inline int msgpack_pack_uint64(msgpack_packer* x, uint64_t d) { msgpack_pack_real_uint64(x, d); } -msgpack_pack_inline_func(_int8)(msgpack_pack_user x, int8_t d) +static inline int msgpack_pack_int8(msgpack_packer* x, int8_t d) { msgpack_pack_real_int8(x, d); } -msgpack_pack_inline_func(_int16)(msgpack_pack_user x, int16_t d) +static inline int msgpack_pack_int16(msgpack_packer* x, int16_t d) { msgpack_pack_real_int16(x, d); } -msgpack_pack_inline_func(_int32)(msgpack_pack_user x, int32_t d) +static inline int msgpack_pack_int32(msgpack_packer* x, int32_t d) { msgpack_pack_real_int32(x, d); } -msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d) +static inline int msgpack_pack_int64(msgpack_packer* x, int64_t d) { msgpack_pack_real_int64(x, d); } @@ -375,7 +307,7 @@ msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d) #ifdef msgpack_pack_inline_func_cint -msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d) +static inline int msgpack_pack_short(msgpack_packer* x, short d) { #if defined(SIZEOF_SHORT) #if SIZEOF_SHORT == 2 @@ -406,7 +338,7 @@ if(sizeof(short) == 2) { #endif } -msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d) +static inline int msgpack_pack_int(msgpack_packer* x, int d) { #if defined(SIZEOF_INT) #if SIZEOF_INT == 2 @@ -437,7 +369,7 @@ if(sizeof(int) == 2) { #endif } -msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d) +static inline int msgpack_pack_long(msgpack_packer* x, long d) { #if defined(SIZEOF_LONG) #if SIZEOF_LONG == 2 @@ -468,7 +400,7 @@ if(sizeof(long) == 2) { #endif } -msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d) +static inline int msgpack_pack_long_long(msgpack_packer* x, long long d) { #if defined(SIZEOF_LONG_LONG) #if SIZEOF_LONG_LONG == 2 @@ -499,7 +431,7 @@ if(sizeof(long long) == 2) { #endif } -msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d) +static inline int msgpack_pack_unsigned_short(msgpack_packer* x, unsigned short d) { #if defined(SIZEOF_SHORT) #if SIZEOF_SHORT == 2 @@ -530,7 +462,7 @@ if(sizeof(unsigned short) == 2) { #endif } -msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d) +static inline int msgpack_pack_unsigned_int(msgpack_packer* x, unsigned int d) { #if defined(SIZEOF_INT) #if SIZEOF_INT == 2 @@ -561,7 +493,7 @@ if(sizeof(unsigned int) == 2) { #endif } -msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d) +static inline int msgpack_pack_unsigned_long(msgpack_packer* x, unsigned long d) { #if defined(SIZEOF_LONG) #if SIZEOF_LONG == 2 @@ -592,7 +524,7 @@ if(sizeof(unsigned long) == 2) { #endif } -msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d) +static inline int msgpack_pack_unsigned_long_long(msgpack_packer* x, unsigned long long d) { #if defined(SIZEOF_LONG_LONG) #if SIZEOF_LONG_LONG == 2 @@ -632,7 +564,7 @@ if(sizeof(unsigned long long) == 2) { * Float */ -msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) +static inline int msgpack_pack_float(msgpack_packer* x, float d) { union { float f; uint32_t i; } mem; mem.f = d; @@ -641,7 +573,7 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) +static inline int msgpack_pack_double(msgpack_packer* x, double d) { union { double f; uint64_t i; } mem; mem.f = d; @@ -660,7 +592,7 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) * Nil */ -msgpack_pack_inline_func(_nil)(msgpack_pack_user x) +static inline int msgpack_pack_nil(msgpack_packer* x) { static const unsigned char d = 0xc0; msgpack_pack_append_buffer(x, &d, 1); @@ -671,13 +603,13 @@ msgpack_pack_inline_func(_nil)(msgpack_pack_user x) * Boolean */ -msgpack_pack_inline_func(_true)(msgpack_pack_user x) +static inline int msgpack_pack_true(msgpack_packer* x) { static const unsigned char d = 0xc3; msgpack_pack_append_buffer(x, &d, 1); } -msgpack_pack_inline_func(_false)(msgpack_pack_user x) +static inline int msgpack_pack_false(msgpack_packer* x) { static const unsigned char d = 0xc2; msgpack_pack_append_buffer(x, &d, 1); @@ -688,7 +620,7 @@ msgpack_pack_inline_func(_false)(msgpack_pack_user x) * Array */ -msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) +static inline int msgpack_pack_array(msgpack_packer* x, unsigned int n) { if(n < 16) { unsigned char d = 0x90 | n; @@ -709,7 +641,7 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) * Map */ -msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) +static inline int msgpack_pack_map(msgpack_packer* x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; @@ -730,7 +662,7 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) * Raw */ -msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) +static inline int msgpack_pack_raw(msgpack_packer* x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | (uint8_t)l; @@ -746,13 +678,11 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) } } -msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l) +static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t l) { msgpack_pack_append_buffer(x, (const unsigned char*)b, l); } -#undef msgpack_pack_inline_func -#undef msgpack_pack_user #undef msgpack_pack_append_buffer #undef TAKE8_8 From c49489cd372903fc5c26b330c91bdd3679029e17 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 27 Feb 2013 21:12:20 +0900 Subject: [PATCH 0694/1172] remove some macros. --- msgpack/_unpacker.pyx | 34 +++++++-------- msgpack/pack.h | 2 +- msgpack/pack_template.h | 6 +-- msgpack/unpack.h | 72 +++++++++++++------------------ msgpack/unpack_template.h | 90 +++++++++++++-------------------------- 5 files changed, 80 insertions(+), 124 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index daeb6d7..5813dc9 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -30,26 +30,26 @@ cdef extern from "unpack.h": char *encoding char *unicode_errors - ctypedef struct template_context: + ctypedef struct unpack_context: msgpack_user user PyObject* obj size_t count unsigned int ct PyObject* key - ctypedef int (*execute_fn)(template_context* ctx, const_char_ptr data, + ctypedef int (*execute_fn)(unpack_context* ctx, const_char_ptr data, size_t len, size_t* off) except? -1 - execute_fn template_construct - execute_fn template_skip + execute_fn unpack_construct + execute_fn unpack_skip execute_fn read_array_header execute_fn read_map_header - void template_init(template_context* ctx) - object template_data(template_context* ctx) + void unpack_init(unpack_context* ctx) + object unpack_data(unpack_context* ctx) -cdef inline init_ctx(template_context *ctx, +cdef inline init_ctx(unpack_context *ctx, object object_hook, object object_pairs_hook, object list_hook, bint use_list, char* encoding, char* unicode_errors): - template_init(ctx) + unpack_init(ctx) ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL @@ -88,7 +88,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, See :class:`Unpacker` for options. """ - cdef template_context ctx + cdef unpack_context ctx cdef size_t off = 0 cdef int ret @@ -110,9 +110,9 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cerr = PyBytes_AsString(unicode_errors) init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) - ret = template_construct(&ctx, buf, buf_len, &off) + ret = unpack_construct(&ctx, buf, buf_len, &off) if ret == 1: - obj = template_data(&ctx) + obj = unpack_data(&ctx) if off < buf_len: raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) return obj @@ -194,7 +194,7 @@ cdef class Unpacker(object): for o in unpacker: process(o) """ - cdef template_context ctx + cdef unpack_context ctx cdef char* buf cdef size_t buf_size, buf_head, buf_tail cdef object file_like @@ -324,8 +324,8 @@ cdef class Unpacker(object): write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head)) if ret == 1: - obj = template_data(&self.ctx) - template_init(&self.ctx) + obj = unpack_data(&self.ctx) + unpack_init(&self.ctx) return obj elif ret == 0: if self.file_like is not None: @@ -357,7 +357,7 @@ cdef class Unpacker(object): Raises `OutOfData` when there are no more bytes to unpack. """ - return self._unpack(template_construct, write_bytes) + return self._unpack(unpack_construct, write_bytes) def skip(self, object write_bytes=None): """ @@ -368,7 +368,7 @@ cdef class Unpacker(object): Raises `OutOfData` when there are no more bytes to unpack. """ - return self._unpack(template_skip, write_bytes) + return self._unpack(unpack_skip, write_bytes) def read_array_header(self, object write_bytes=None): """assuming the next object is an array, return its size n, such that @@ -390,7 +390,7 @@ cdef class Unpacker(object): return self def __next__(self): - return self._unpack(template_construct, None, 1) + return self._unpack(unpack_construct, None, 1) # for debug. #def _buf(self): diff --git a/msgpack/pack.h b/msgpack/pack.h index 59764d9..1539991 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -45,7 +45,7 @@ static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d); static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); -static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); +//static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index d6bf820..35deb83 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -305,7 +305,7 @@ static inline int msgpack_pack_int64(msgpack_packer* x, int64_t d) } -#ifdef msgpack_pack_inline_func_cint +//#ifdef msgpack_pack_inline_func_cint static inline int msgpack_pack_short(msgpack_packer* x, short d) { @@ -555,8 +555,8 @@ if(sizeof(unsigned long long) == 2) { #endif } -#undef msgpack_pack_inline_func_cint -#endif +//#undef msgpack_pack_inline_func_cint +//#endif diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 3dc88e5..595b8df 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -28,31 +28,17 @@ typedef struct unpack_user { const char *unicode_errors; } unpack_user; +typedef PyObject* msgpack_unpack_object; +struct unpack_context; +typedef struct unpack_context unpack_context; +typedef int (*execute_fn)(unpack_context *ctx, const char* data, size_t len, size_t* off); -#define msgpack_unpack_struct(name) \ - struct template ## name - -#define msgpack_unpack_func(ret, name) \ - static inline ret template ## name - -#define msgpack_unpack_callback(name) \ - template_callback ## name - -#define msgpack_unpack_object PyObject* - -#define msgpack_unpack_user unpack_user - -typedef int (*execute_fn)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off); - -struct template_context; -typedef struct template_context template_context; - -static inline msgpack_unpack_object template_callback_root(unpack_user* u) +static inline msgpack_unpack_object unpack_callback_root(unpack_user* u) { return NULL; } -static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) +static inline int unpack_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) { PyObject *p = PyInt_FromLong((long)d); if (!p) @@ -60,13 +46,13 @@ static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_u *o = p; return 0; } -static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) +static inline int unpack_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) { - return template_callback_uint16(u, d, o); + return unpack_callback_uint16(u, d, o); } -static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) +static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { PyObject *p; if (d > LONG_MAX) { @@ -80,7 +66,7 @@ static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_u return 0; } -static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) +static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) { PyObject *p = PyLong_FromUnsignedLongLong(d); if (!p) @@ -89,7 +75,7 @@ static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_u return 0; } -static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) +static inline int unpack_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) { PyObject *p = PyInt_FromLong(d); if (!p) @@ -98,17 +84,17 @@ static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unp return 0; } -static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) +static inline int unpack_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) { - return template_callback_int32(u, d, o); + return unpack_callback_int32(u, d, o); } -static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +static inline int unpack_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) { - return template_callback_int32(u, d, o); + return unpack_callback_int32(u, d, o); } -static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) +static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) { PyObject *p = PyLong_FromLongLong(d); if (!p) @@ -117,7 +103,7 @@ static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unp return 0; } -static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) +static inline int unpack_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) { PyObject *p = PyFloat_FromDouble(d); if (!p) @@ -126,21 +112,21 @@ static inline int template_callback_double(unpack_user* u, double d, msgpack_unp return 0; } -static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +static inline int unpack_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) { - return template_callback_double(u, d, o); + return unpack_callback_double(u, d, o); } -static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) +static inline int unpack_callback_nil(unpack_user* u, msgpack_unpack_object* o) { Py_INCREF(Py_None); *o = Py_None; return 0; } -static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o) +static inline int unpack_callback_true(unpack_user* u, msgpack_unpack_object* o) { Py_INCREF(Py_True); *o = Py_True; return 0; } -static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o) +static inline int unpack_callback_false(unpack_user* u, msgpack_unpack_object* o) { Py_INCREF(Py_False); *o = Py_False; return 0; } -static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +static inline int unpack_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { PyObject *p = u->use_list ? PyList_New(n) : PyTuple_New(n); @@ -150,7 +136,7 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac return 0; } -static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) +static inline int unpack_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) { if (u->use_list) PyList_SET_ITEM(*c, current, o); @@ -159,7 +145,7 @@ static inline int template_callback_array_item(unpack_user* u, unsigned int curr return 0; } -static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_object* c) +static inline int unpack_callback_array_end(unpack_user* u, msgpack_unpack_object* c) { if (u->list_hook) { PyObject *new_c = PyEval_CallFunction(u->list_hook, "(O)", *c); @@ -171,7 +157,7 @@ static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_obj return 0; } -static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +static inline int unpack_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { PyObject *p; if (u->has_pairs_hook) { @@ -186,7 +172,7 @@ static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_ return 0; } -static inline int template_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) +static inline int unpack_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { if (u->has_pairs_hook) { msgpack_unpack_object item = PyTuple_Pack(2, k, v); @@ -205,7 +191,7 @@ static inline int template_callback_map_item(unpack_user* u, unsigned int curren return -1; } -static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c) +static inline int unpack_callback_map_end(unpack_user* u, msgpack_unpack_object* c) { if (u->object_hook) { PyObject *new_c = PyEval_CallFunction(u->object_hook, "(O)", *c); @@ -218,7 +204,7 @@ static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_objec return 0; } -static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) +static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { PyObject *py; if(u->encoding) { diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 83b6918..153340c 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -16,59 +16,35 @@ * limitations under the License. */ -#ifndef msgpack_unpack_func -#error msgpack_unpack_func template is not defined -#endif - -#ifndef msgpack_unpack_callback -#error msgpack_unpack_callback template is not defined -#endif - -#ifndef msgpack_unpack_struct -#error msgpack_unpack_struct template is not defined -#endif - -#ifndef msgpack_unpack_struct_decl -#define msgpack_unpack_struct_decl(name) msgpack_unpack_struct(name) -#endif - -#ifndef msgpack_unpack_object -#error msgpack_unpack_object type is not defined -#endif - -#ifndef msgpack_unpack_user -#error msgpack_unpack_user type is not defined -#endif - #ifndef USE_CASE_RANGE #if !defined(_MSC_VER) #define USE_CASE_RANGE #endif #endif -msgpack_unpack_struct_decl(_stack) { - msgpack_unpack_object obj; +typedef struct unpack_stack { + PyObject* obj; size_t size; size_t count; unsigned int ct; - msgpack_unpack_object map_key; -}; + PyObject* map_key; +} unpack_stack; -msgpack_unpack_struct_decl(_context) { - msgpack_unpack_user user; +struct unpack_context { + unpack_user user; unsigned int cs; unsigned int trail; unsigned int top; /* - msgpack_unpack_struct(_stack)* stack; + unpack_stack* stack; unsigned int stack_size; - msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE]; + unpack_stack embed_stack[MSGPACK_EMBED_STACK_SIZE]; */ - msgpack_unpack_struct(_stack) stack[MSGPACK_EMBED_STACK_SIZE]; + unpack_stack stack[MSGPACK_EMBED_STACK_SIZE]; }; -msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) +static inline void unpack_init(unpack_context* ctx) { ctx->cs = CS_HEADER; ctx->trail = 0; @@ -77,11 +53,11 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) ctx->stack = ctx->embed_stack; ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; */ - ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); + ctx->stack[0].obj = unpack_callback_root(&ctx->user); } /* -msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx) +static inline void unpack_destroy(unpack_context* ctx) { if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { free(ctx->stack); @@ -89,14 +65,14 @@ msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx) } */ -msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) +static inline PyObject* unpack_data(unpack_context* ctx) { return (ctx)->stack[0].obj; } template -msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) +static inline int unpack_execute(unpack_context* ctx, const char* data, size_t len, size_t* off) { assert(len >= *off); @@ -107,19 +83,19 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c unsigned int trail = ctx->trail; unsigned int cs = ctx->cs; unsigned int top = ctx->top; - msgpack_unpack_struct(_stack)* stack = ctx->stack; + unpack_stack* stack = ctx->stack; /* unsigned int stack_size = ctx->stack_size; */ - msgpack_unpack_user* user = &ctx->user; + unpack_user* user = &ctx->user; - msgpack_unpack_object obj; - msgpack_unpack_struct(_stack)* c = NULL; + PyObject* obj; + unpack_stack* c = NULL; int ret; #define construct_cb(name) \ - construct && msgpack_unpack_callback(name) + construct && unpack_callback ## name #define push_simple_value(func) \ if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \ @@ -157,16 +133,16 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c /* FIXME \ if(top >= stack_size) { \ if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \ - size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \ + size_t csize = sizeof(unpack_stack) * MSGPACK_EMBED_STACK_SIZE; \ size_t nsize = csize * 2; \ - msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \ + unpack_stack* tmp = (unpack_stack*)malloc(nsize); \ if(tmp == NULL) { goto _failed; } \ memcpy(tmp, ctx->stack, csize); \ ctx->stack = stack = tmp; \ ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ } else { \ - size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \ - msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \ + size_t nsize = sizeof(unpack_stack) * ctx->stack_size * 2; \ + unpack_stack* tmp = (unpack_stack*)realloc(ctx->stack, nsize); \ if(tmp == NULL) { goto _failed; } \ ctx->stack = stack = tmp; \ ctx->stack_size = stack_size = stack_size * 2; \ @@ -382,7 +358,7 @@ _header_again: _finish: if (!construct) - msgpack_unpack_callback(_nil)(user, &obj); + unpack_callback_nil(user, &obj); stack[0].obj = obj; ++p; ret = 1; @@ -420,7 +396,7 @@ _end: #undef start_container template -msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) +static inline int unpack_container_header(unpack_context* ctx, const char* data, size_t len, size_t* off) { assert(len >= *off); uint32_t size; @@ -467,7 +443,7 @@ msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); return -1; } - msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj); + unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj); return 1; } @@ -476,16 +452,10 @@ msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx #undef SWITCH_RANGE_DEFAULT #undef SWITCH_RANGE_END -static const execute_fn template_construct = &template_execute; -static const execute_fn template_skip = &template_execute; -static const execute_fn read_array_header = &template_container_header<0x90, 0xdc>; -static const execute_fn read_map_header = &template_container_header<0x80, 0xde>; - -#undef msgpack_unpack_func -#undef msgpack_unpack_callback -#undef msgpack_unpack_struct -#undef msgpack_unpack_object -#undef msgpack_unpack_user +static const execute_fn unpack_construct = &unpack_execute; +static const execute_fn unpack_skip = &unpack_execute; +static const execute_fn read_array_header = &unpack_container_header<0x90, 0xdc>; +static const execute_fn read_map_header = &unpack_container_header<0x80, 0xde>; #undef NEXT_CS From c9b6e5b65d8cbc8a1ed5450eb274326e1c061406 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 27 Feb 2013 21:24:25 +0900 Subject: [PATCH 0695/1172] s/\t/ /g --- msgpack/pack_template.h | 655 +++++++++++++++++++------------------- msgpack/sysdep.h | 1 - msgpack/unpack_define.h | 76 ++--- msgpack/unpack_template.h | 628 ++++++++++++++++++------------------ 4 files changed, 679 insertions(+), 681 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 35deb83..9e00d7e 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -39,269 +39,269 @@ #define msgpack_pack_real_uint8(x, d) \ do { \ - if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ - } else { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ } while(0) #define msgpack_pack_real_uint16(x, d) \ do { \ - if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ - } else if(d < (1<<8)) { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } else { \ - /* unsigned 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ + } else if(d < (1<<8)) { \ + /* unsigned 8 */ \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + unsigned char buf[3]; \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ } while(0) #define msgpack_pack_real_uint32(x, d) \ do { \ - if(d < (1<<8)) { \ - if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ - } else { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ - } else { \ - if(d < (1<<16)) { \ - /* unsigned 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else { \ - /* unsigned 32 */ \ - unsigned char buf[5]; \ - buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ - msgpack_pack_append_buffer(x, buf, 5); \ - } \ - } \ + if(d < (1<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ + } else { \ + /* unsigned 8 */ \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1<<16)) { \ + /* unsigned 16 */ \ + unsigned char buf[3]; \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + unsigned char buf[5]; \ + buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ } while(0) #define msgpack_pack_real_uint64(x, d) \ do { \ - if(d < (1ULL<<8)) { \ - if(d < (1ULL<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ - } else { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ - } else { \ - if(d < (1ULL<<16)) { \ - /* unsigned 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else if(d < (1ULL<<32)) { \ - /* unsigned 32 */ \ - unsigned char buf[5]; \ - buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ - msgpack_pack_append_buffer(x, buf, 5); \ - } else { \ - /* unsigned 64 */ \ - unsigned char buf[9]; \ - buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \ - msgpack_pack_append_buffer(x, buf, 9); \ - } \ - } \ + if(d < (1ULL<<8)) { \ + if(d < (1ULL<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ + } else { \ + /* unsigned 8 */ \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1ULL<<16)) { \ + /* unsigned 16 */ \ + unsigned char buf[3]; \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else if(d < (1ULL<<32)) { \ + /* unsigned 32 */ \ + unsigned char buf[5]; \ + buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* unsigned 64 */ \ + unsigned char buf[9]; \ + buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ } while(0) #define msgpack_pack_real_int8(x, d) \ do { \ - if(d < -(1<<5)) { \ - /* signed 8 */ \ - unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } else { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ - } \ + if(d < -(1<<5)) { \ + /* signed 8 */ \ + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ + } \ } while(0) #define msgpack_pack_real_int16(x, d) \ do { \ - if(d < -(1<<5)) { \ - if(d < -(1<<7)) { \ - /* signed 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else { \ - /* signed 8 */ \ - unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ - } else if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ - } else { \ - if(d < (1<<8)) { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } else { \ - /* unsigned 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } \ - } \ + if(d < -(1<<5)) { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + unsigned char buf[3]; \ + buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + unsigned char buf[3]; \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } \ } while(0) #define msgpack_pack_real_int32(x, d) \ do { \ - if(d < -(1<<5)) { \ - if(d < -(1<<15)) { \ - /* signed 32 */ \ - unsigned char buf[5]; \ - buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \ - msgpack_pack_append_buffer(x, buf, 5); \ - } else if(d < -(1<<7)) { \ - /* signed 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else { \ - /* signed 8 */ \ - unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ - } else if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ - } else { \ - if(d < (1<<8)) { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } else if(d < (1<<16)) { \ - /* unsigned 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else { \ - /* unsigned 32 */ \ - unsigned char buf[5]; \ - buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ - msgpack_pack_append_buffer(x, buf, 5); \ - } \ - } \ + if(d < -(1<<5)) { \ + if(d < -(1<<15)) { \ + /* signed 32 */ \ + unsigned char buf[5]; \ + buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else if(d < -(1<<7)) { \ + /* signed 16 */ \ + unsigned char buf[3]; \ + buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else if(d < (1<<16)) { \ + /* unsigned 16 */ \ + unsigned char buf[3]; \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + unsigned char buf[5]; \ + buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ } while(0) #define msgpack_pack_real_int64(x, d) \ do { \ - if(d < -(1LL<<5)) { \ - if(d < -(1LL<<15)) { \ - if(d < -(1LL<<31)) { \ - /* signed 64 */ \ - unsigned char buf[9]; \ - buf[0] = 0xd3; _msgpack_store64(&buf[1], d); \ - msgpack_pack_append_buffer(x, buf, 9); \ - } else { \ - /* signed 32 */ \ - unsigned char buf[5]; \ - buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \ - msgpack_pack_append_buffer(x, buf, 5); \ - } \ - } else { \ - if(d < -(1<<7)) { \ - /* signed 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else { \ - /* signed 8 */ \ - unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ - } \ - } else if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ - } else { \ - if(d < (1LL<<16)) { \ - if(d < (1<<8)) { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } else { \ - /* unsigned 16 */ \ - unsigned char buf[3]; \ - buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ - msgpack_pack_append_buffer(x, buf, 3); \ - } \ - } else { \ - if(d < (1LL<<32)) { \ - /* unsigned 32 */ \ - unsigned char buf[5]; \ - buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ - msgpack_pack_append_buffer(x, buf, 5); \ - } else { \ - /* unsigned 64 */ \ - unsigned char buf[9]; \ - buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \ - msgpack_pack_append_buffer(x, buf, 9); \ - } \ - } \ - } \ + if(d < -(1LL<<5)) { \ + if(d < -(1LL<<15)) { \ + if(d < -(1LL<<31)) { \ + /* signed 64 */ \ + unsigned char buf[9]; \ + buf[0] = 0xd3; _msgpack_store64(&buf[1], d); \ + msgpack_pack_append_buffer(x, buf, 9); \ + } else { \ + /* signed 32 */ \ + unsigned char buf[5]; \ + buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } else { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + unsigned char buf[3]; \ + buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ + } else { \ + if(d < (1LL<<16)) { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + unsigned char buf[3]; \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } else { \ + if(d < (1LL<<32)) { \ + /* unsigned 32 */ \ + unsigned char buf[5]; \ + buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* unsigned 64 */ \ + unsigned char buf[9]; \ + buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ + } \ } while(0) static inline int msgpack_pack_uint8(msgpack_packer* x, uint8_t d) { - msgpack_pack_real_uint8(x, d); + msgpack_pack_real_uint8(x, d); } static inline int msgpack_pack_uint16(msgpack_packer* x, uint16_t d) { - msgpack_pack_real_uint16(x, d); + msgpack_pack_real_uint16(x, d); } static inline int msgpack_pack_uint32(msgpack_packer* x, uint32_t d) { - msgpack_pack_real_uint32(x, d); + msgpack_pack_real_uint32(x, d); } static inline int msgpack_pack_uint64(msgpack_packer* x, uint64_t d) { - msgpack_pack_real_uint64(x, d); + msgpack_pack_real_uint64(x, d); } static inline int msgpack_pack_int8(msgpack_packer* x, int8_t d) { - msgpack_pack_real_int8(x, d); + msgpack_pack_real_int8(x, d); } static inline int msgpack_pack_int16(msgpack_packer* x, int16_t d) { - msgpack_pack_real_int16(x, d); + msgpack_pack_real_int16(x, d); } static inline int msgpack_pack_int32(msgpack_packer* x, int32_t d) { - msgpack_pack_real_int32(x, d); + msgpack_pack_real_int32(x, d); } static inline int msgpack_pack_int64(msgpack_packer* x, int64_t d) { - msgpack_pack_real_int64(x, d); + msgpack_pack_real_int64(x, d); } @@ -311,29 +311,29 @@ static inline int msgpack_pack_short(msgpack_packer* x, short d) { #if defined(SIZEOF_SHORT) #if SIZEOF_SHORT == 2 - msgpack_pack_real_int16(x, d); + msgpack_pack_real_int16(x, d); #elif SIZEOF_SHORT == 4 - msgpack_pack_real_int32(x, d); + msgpack_pack_real_int32(x, d); #else - msgpack_pack_real_int64(x, d); + msgpack_pack_real_int64(x, d); #endif #elif defined(SHRT_MAX) #if SHRT_MAX == 0x7fff - msgpack_pack_real_int16(x, d); + msgpack_pack_real_int16(x, d); #elif SHRT_MAX == 0x7fffffff - msgpack_pack_real_int32(x, d); + msgpack_pack_real_int32(x, d); #else - msgpack_pack_real_int64(x, d); + msgpack_pack_real_int64(x, d); #endif #else if(sizeof(short) == 2) { - msgpack_pack_real_int16(x, d); + msgpack_pack_real_int16(x, d); } else if(sizeof(short) == 4) { - msgpack_pack_real_int32(x, d); + msgpack_pack_real_int32(x, d); } else { - msgpack_pack_real_int64(x, d); + msgpack_pack_real_int64(x, d); } #endif } @@ -342,29 +342,29 @@ static inline int msgpack_pack_int(msgpack_packer* x, int d) { #if defined(SIZEOF_INT) #if SIZEOF_INT == 2 - msgpack_pack_real_int16(x, d); + msgpack_pack_real_int16(x, d); #elif SIZEOF_INT == 4 - msgpack_pack_real_int32(x, d); + msgpack_pack_real_int32(x, d); #else - msgpack_pack_real_int64(x, d); + msgpack_pack_real_int64(x, d); #endif #elif defined(INT_MAX) #if INT_MAX == 0x7fff - msgpack_pack_real_int16(x, d); + msgpack_pack_real_int16(x, d); #elif INT_MAX == 0x7fffffff - msgpack_pack_real_int32(x, d); + msgpack_pack_real_int32(x, d); #else - msgpack_pack_real_int64(x, d); + msgpack_pack_real_int64(x, d); #endif #else if(sizeof(int) == 2) { - msgpack_pack_real_int16(x, d); + msgpack_pack_real_int16(x, d); } else if(sizeof(int) == 4) { - msgpack_pack_real_int32(x, d); + msgpack_pack_real_int32(x, d); } else { - msgpack_pack_real_int64(x, d); + msgpack_pack_real_int64(x, d); } #endif } @@ -373,29 +373,29 @@ static inline int msgpack_pack_long(msgpack_packer* x, long d) { #if defined(SIZEOF_LONG) #if SIZEOF_LONG == 2 - msgpack_pack_real_int16(x, d); + msgpack_pack_real_int16(x, d); #elif SIZEOF_LONG == 4 - msgpack_pack_real_int32(x, d); + msgpack_pack_real_int32(x, d); #else - msgpack_pack_real_int64(x, d); + msgpack_pack_real_int64(x, d); #endif #elif defined(LONG_MAX) #if LONG_MAX == 0x7fffL - msgpack_pack_real_int16(x, d); + msgpack_pack_real_int16(x, d); #elif LONG_MAX == 0x7fffffffL - msgpack_pack_real_int32(x, d); + msgpack_pack_real_int32(x, d); #else - msgpack_pack_real_int64(x, d); + msgpack_pack_real_int64(x, d); #endif #else if(sizeof(long) == 2) { - msgpack_pack_real_int16(x, d); + msgpack_pack_real_int16(x, d); } else if(sizeof(long) == 4) { - msgpack_pack_real_int32(x, d); + msgpack_pack_real_int32(x, d); } else { - msgpack_pack_real_int64(x, d); + msgpack_pack_real_int64(x, d); } #endif } @@ -404,29 +404,29 @@ static inline int msgpack_pack_long_long(msgpack_packer* x, long long d) { #if defined(SIZEOF_LONG_LONG) #if SIZEOF_LONG_LONG == 2 - msgpack_pack_real_int16(x, d); + msgpack_pack_real_int16(x, d); #elif SIZEOF_LONG_LONG == 4 - msgpack_pack_real_int32(x, d); + msgpack_pack_real_int32(x, d); #else - msgpack_pack_real_int64(x, d); + msgpack_pack_real_int64(x, d); #endif #elif defined(LLONG_MAX) #if LLONG_MAX == 0x7fffL - msgpack_pack_real_int16(x, d); + msgpack_pack_real_int16(x, d); #elif LLONG_MAX == 0x7fffffffL - msgpack_pack_real_int32(x, d); + msgpack_pack_real_int32(x, d); #else - msgpack_pack_real_int64(x, d); + msgpack_pack_real_int64(x, d); #endif #else if(sizeof(long long) == 2) { - msgpack_pack_real_int16(x, d); + msgpack_pack_real_int16(x, d); } else if(sizeof(long long) == 4) { - msgpack_pack_real_int32(x, d); + msgpack_pack_real_int32(x, d); } else { - msgpack_pack_real_int64(x, d); + msgpack_pack_real_int64(x, d); } #endif } @@ -435,29 +435,29 @@ static inline int msgpack_pack_unsigned_short(msgpack_packer* x, unsigned short { #if defined(SIZEOF_SHORT) #if SIZEOF_SHORT == 2 - msgpack_pack_real_uint16(x, d); + msgpack_pack_real_uint16(x, d); #elif SIZEOF_SHORT == 4 - msgpack_pack_real_uint32(x, d); + msgpack_pack_real_uint32(x, d); #else - msgpack_pack_real_uint64(x, d); + msgpack_pack_real_uint64(x, d); #endif #elif defined(USHRT_MAX) #if USHRT_MAX == 0xffffU - msgpack_pack_real_uint16(x, d); + msgpack_pack_real_uint16(x, d); #elif USHRT_MAX == 0xffffffffU - msgpack_pack_real_uint32(x, d); + msgpack_pack_real_uint32(x, d); #else - msgpack_pack_real_uint64(x, d); + msgpack_pack_real_uint64(x, d); #endif #else if(sizeof(unsigned short) == 2) { - msgpack_pack_real_uint16(x, d); + msgpack_pack_real_uint16(x, d); } else if(sizeof(unsigned short) == 4) { - msgpack_pack_real_uint32(x, d); + msgpack_pack_real_uint32(x, d); } else { - msgpack_pack_real_uint64(x, d); + msgpack_pack_real_uint64(x, d); } #endif } @@ -466,29 +466,29 @@ static inline int msgpack_pack_unsigned_int(msgpack_packer* x, unsigned int d) { #if defined(SIZEOF_INT) #if SIZEOF_INT == 2 - msgpack_pack_real_uint16(x, d); + msgpack_pack_real_uint16(x, d); #elif SIZEOF_INT == 4 - msgpack_pack_real_uint32(x, d); + msgpack_pack_real_uint32(x, d); #else - msgpack_pack_real_uint64(x, d); + msgpack_pack_real_uint64(x, d); #endif #elif defined(UINT_MAX) #if UINT_MAX == 0xffffU - msgpack_pack_real_uint16(x, d); + msgpack_pack_real_uint16(x, d); #elif UINT_MAX == 0xffffffffU - msgpack_pack_real_uint32(x, d); + msgpack_pack_real_uint32(x, d); #else - msgpack_pack_real_uint64(x, d); + msgpack_pack_real_uint64(x, d); #endif #else if(sizeof(unsigned int) == 2) { - msgpack_pack_real_uint16(x, d); + msgpack_pack_real_uint16(x, d); } else if(sizeof(unsigned int) == 4) { - msgpack_pack_real_uint32(x, d); + msgpack_pack_real_uint32(x, d); } else { - msgpack_pack_real_uint64(x, d); + msgpack_pack_real_uint64(x, d); } #endif } @@ -497,29 +497,29 @@ static inline int msgpack_pack_unsigned_long(msgpack_packer* x, unsigned long d) { #if defined(SIZEOF_LONG) #if SIZEOF_LONG == 2 - msgpack_pack_real_uint16(x, d); + msgpack_pack_real_uint16(x, d); #elif SIZEOF_LONG == 4 - msgpack_pack_real_uint32(x, d); + msgpack_pack_real_uint32(x, d); #else - msgpack_pack_real_uint64(x, d); + msgpack_pack_real_uint64(x, d); #endif #elif defined(ULONG_MAX) #if ULONG_MAX == 0xffffUL - msgpack_pack_real_uint16(x, d); + msgpack_pack_real_uint16(x, d); #elif ULONG_MAX == 0xffffffffUL - msgpack_pack_real_uint32(x, d); + msgpack_pack_real_uint32(x, d); #else - msgpack_pack_real_uint64(x, d); + msgpack_pack_real_uint64(x, d); #endif #else if(sizeof(unsigned long) == 2) { - msgpack_pack_real_uint16(x, d); + msgpack_pack_real_uint16(x, d); } else if(sizeof(unsigned long) == 4) { - msgpack_pack_real_uint32(x, d); + msgpack_pack_real_uint32(x, d); } else { - msgpack_pack_real_uint64(x, d); + msgpack_pack_real_uint64(x, d); } #endif } @@ -528,29 +528,29 @@ static inline int msgpack_pack_unsigned_long_long(msgpack_packer* x, unsigned lo { #if defined(SIZEOF_LONG_LONG) #if SIZEOF_LONG_LONG == 2 - msgpack_pack_real_uint16(x, d); + msgpack_pack_real_uint16(x, d); #elif SIZEOF_LONG_LONG == 4 - msgpack_pack_real_uint32(x, d); + msgpack_pack_real_uint32(x, d); #else - msgpack_pack_real_uint64(x, d); + msgpack_pack_real_uint64(x, d); #endif #elif defined(ULLONG_MAX) #if ULLONG_MAX == 0xffffUL - msgpack_pack_real_uint16(x, d); + msgpack_pack_real_uint16(x, d); #elif ULLONG_MAX == 0xffffffffUL - msgpack_pack_real_uint32(x, d); + msgpack_pack_real_uint32(x, d); #else - msgpack_pack_real_uint64(x, d); + msgpack_pack_real_uint64(x, d); #endif #else if(sizeof(unsigned long long) == 2) { - msgpack_pack_real_uint16(x, d); + msgpack_pack_real_uint16(x, d); } else if(sizeof(unsigned long long) == 4) { - msgpack_pack_real_uint32(x, d); + msgpack_pack_real_uint32(x, d); } else { - msgpack_pack_real_uint64(x, d); + msgpack_pack_real_uint64(x, d); } #endif } @@ -566,25 +566,25 @@ if(sizeof(unsigned long long) == 2) { static inline int msgpack_pack_float(msgpack_packer* x, float d) { - union { float f; uint32_t i; } mem; - mem.f = d; - unsigned char buf[5]; - buf[0] = 0xca; _msgpack_store32(&buf[1], mem.i); - msgpack_pack_append_buffer(x, buf, 5); + union { float f; uint32_t i; } mem; + mem.f = d; + unsigned char buf[5]; + buf[0] = 0xca; _msgpack_store32(&buf[1], mem.i); + msgpack_pack_append_buffer(x, buf, 5); } static inline int msgpack_pack_double(msgpack_packer* x, double d) { - union { double f; uint64_t i; } mem; - mem.f = d; - unsigned char buf[9]; - buf[0] = 0xcb; + union { double f; uint64_t i; } mem; + mem.f = d; + unsigned char buf[9]; + buf[0] = 0xcb; #if defined(__arm__) && !(__ARM_EABI__) // arm-oabi // https://github.com/msgpack/msgpack-perl/pull/1 mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL); #endif _msgpack_store64(&buf[1], mem.i); - msgpack_pack_append_buffer(x, buf, 9); + msgpack_pack_append_buffer(x, buf, 9); } @@ -594,8 +594,8 @@ static inline int msgpack_pack_double(msgpack_packer* x, double d) static inline int msgpack_pack_nil(msgpack_packer* x) { - static const unsigned char d = 0xc0; - msgpack_pack_append_buffer(x, &d, 1); + static const unsigned char d = 0xc0; + msgpack_pack_append_buffer(x, &d, 1); } @@ -605,14 +605,14 @@ static inline int msgpack_pack_nil(msgpack_packer* x) static inline int msgpack_pack_true(msgpack_packer* x) { - static const unsigned char d = 0xc3; - msgpack_pack_append_buffer(x, &d, 1); + static const unsigned char d = 0xc3; + msgpack_pack_append_buffer(x, &d, 1); } static inline int msgpack_pack_false(msgpack_packer* x) { - static const unsigned char d = 0xc2; - msgpack_pack_append_buffer(x, &d, 1); + static const unsigned char d = 0xc2; + msgpack_pack_append_buffer(x, &d, 1); } @@ -622,18 +622,18 @@ static inline int msgpack_pack_false(msgpack_packer* x) static inline int msgpack_pack_array(msgpack_packer* x, unsigned int n) { - if(n < 16) { - unsigned char d = 0x90 | n; - msgpack_pack_append_buffer(x, &d, 1); - } else if(n < 65536) { - unsigned char buf[3]; - buf[0] = 0xdc; _msgpack_store16(&buf[1], (uint16_t)n); - msgpack_pack_append_buffer(x, buf, 3); - } else { - unsigned char buf[5]; - buf[0] = 0xdd; _msgpack_store32(&buf[1], (uint32_t)n); - msgpack_pack_append_buffer(x, buf, 5); - } + if(n < 16) { + unsigned char d = 0x90 | n; + msgpack_pack_append_buffer(x, &d, 1); + } else if(n < 65536) { + unsigned char buf[3]; + buf[0] = 0xdc; _msgpack_store16(&buf[1], (uint16_t)n); + msgpack_pack_append_buffer(x, buf, 3); + } else { + unsigned char buf[5]; + buf[0] = 0xdd; _msgpack_store32(&buf[1], (uint32_t)n); + msgpack_pack_append_buffer(x, buf, 5); + } } @@ -643,18 +643,18 @@ static inline int msgpack_pack_array(msgpack_packer* x, unsigned int n) static inline int msgpack_pack_map(msgpack_packer* x, unsigned int n) { - if(n < 16) { - unsigned char d = 0x80 | n; - msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); - } else if(n < 65536) { - unsigned char buf[3]; - buf[0] = 0xde; _msgpack_store16(&buf[1], (uint16_t)n); - msgpack_pack_append_buffer(x, buf, 3); - } else { - unsigned char buf[5]; - buf[0] = 0xdf; _msgpack_store32(&buf[1], (uint32_t)n); - msgpack_pack_append_buffer(x, buf, 5); - } + if(n < 16) { + unsigned char d = 0x80 | n; + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); + } else if(n < 65536) { + unsigned char buf[3]; + buf[0] = 0xde; _msgpack_store16(&buf[1], (uint16_t)n); + msgpack_pack_append_buffer(x, buf, 3); + } else { + unsigned char buf[5]; + buf[0] = 0xdf; _msgpack_store32(&buf[1], (uint32_t)n); + msgpack_pack_append_buffer(x, buf, 5); + } } @@ -664,23 +664,23 @@ static inline int msgpack_pack_map(msgpack_packer* x, unsigned int n) static inline int msgpack_pack_raw(msgpack_packer* x, size_t l) { - if(l < 32) { - unsigned char d = 0xa0 | (uint8_t)l; - msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); - } else if(l < 65536) { - unsigned char buf[3]; - buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l); - msgpack_pack_append_buffer(x, buf, 3); - } else { - unsigned char buf[5]; - buf[0] = 0xdb; _msgpack_store32(&buf[1], (uint32_t)l); - msgpack_pack_append_buffer(x, buf, 5); - } + if(l < 32) { + unsigned char d = 0xa0 | (uint8_t)l; + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); + } else if(l < 65536) { + unsigned char buf[3]; + buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l); + msgpack_pack_append_buffer(x, buf, 3); + } else { + unsigned char buf[5]; + buf[0] = 0xdb; _msgpack_store32(&buf[1], (uint32_t)l); + msgpack_pack_append_buffer(x, buf, 5); + } } static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t l) { - msgpack_pack_append_buffer(x, (const unsigned char*)b, l); + msgpack_pack_append_buffer(x, (const unsigned char*)b, l); } #undef msgpack_pack_append_buffer @@ -698,4 +698,3 @@ static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t #undef msgpack_pack_real_int16 #undef msgpack_pack_real_int32 #undef msgpack_pack_real_int64 - diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h index 4fedbd8..ed9c1bc 100644 --- a/msgpack/sysdep.h +++ b/msgpack/sysdep.h @@ -192,4 +192,3 @@ typedef unsigned int _msgpack_atomic_counter_t; #endif /* msgpack/sysdep.h */ - diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 959d351..c81b990 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -35,53 +35,53 @@ extern "C" { typedef enum { - CS_HEADER = 0x00, // nil + CS_HEADER = 0x00, // nil - //CS_ = 0x01, - //CS_ = 0x02, // false - //CS_ = 0x03, // true + //CS_ = 0x01, + //CS_ = 0x02, // false + //CS_ = 0x03, // true - //CS_ = 0x04, - //CS_ = 0x05, - //CS_ = 0x06, - //CS_ = 0x07, + //CS_ = 0x04, + //CS_ = 0x05, + //CS_ = 0x06, + //CS_ = 0x07, - //CS_ = 0x08, - //CS_ = 0x09, - CS_FLOAT = 0x0a, - CS_DOUBLE = 0x0b, - CS_UINT_8 = 0x0c, - CS_UINT_16 = 0x0d, - CS_UINT_32 = 0x0e, - CS_UINT_64 = 0x0f, - CS_INT_8 = 0x10, - CS_INT_16 = 0x11, - CS_INT_32 = 0x12, - CS_INT_64 = 0x13, + //CS_ = 0x08, + //CS_ = 0x09, + CS_FLOAT = 0x0a, + CS_DOUBLE = 0x0b, + CS_UINT_8 = 0x0c, + CS_UINT_16 = 0x0d, + CS_UINT_32 = 0x0e, + CS_UINT_64 = 0x0f, + CS_INT_8 = 0x10, + CS_INT_16 = 0x11, + CS_INT_32 = 0x12, + CS_INT_64 = 0x13, - //CS_ = 0x14, - //CS_ = 0x15, - //CS_BIG_INT_16 = 0x16, - //CS_BIG_INT_32 = 0x17, - //CS_BIG_FLOAT_16 = 0x18, - //CS_BIG_FLOAT_32 = 0x19, - CS_RAW_16 = 0x1a, - CS_RAW_32 = 0x1b, - CS_ARRAY_16 = 0x1c, - CS_ARRAY_32 = 0x1d, - CS_MAP_16 = 0x1e, - CS_MAP_32 = 0x1f, + //CS_ = 0x14, + //CS_ = 0x15, + //CS_BIG_INT_16 = 0x16, + //CS_BIG_INT_32 = 0x17, + //CS_BIG_FLOAT_16 = 0x18, + //CS_BIG_FLOAT_32 = 0x19, + CS_RAW_16 = 0x1a, + CS_RAW_32 = 0x1b, + CS_ARRAY_16 = 0x1c, + CS_ARRAY_32 = 0x1d, + CS_MAP_16 = 0x1e, + CS_MAP_32 = 0x1f, - //ACS_BIG_INT_VALUE, - //ACS_BIG_FLOAT_VALUE, - ACS_RAW_VALUE, + //ACS_BIG_INT_VALUE, + //ACS_BIG_FLOAT_VALUE, + ACS_RAW_VALUE, } msgpack_unpack_state; typedef enum { - CT_ARRAY_ITEM, - CT_MAP_KEY, - CT_MAP_VALUE, + CT_ARRAY_ITEM, + CT_MAP_KEY, + CT_MAP_VALUE, } msgpack_container_type; diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 153340c..29ac935 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -23,136 +23,136 @@ #endif typedef struct unpack_stack { - PyObject* obj; - size_t size; - size_t count; - unsigned int ct; - PyObject* map_key; + PyObject* obj; + size_t size; + size_t count; + unsigned int ct; + PyObject* map_key; } unpack_stack; struct unpack_context { - unpack_user user; - unsigned int cs; - unsigned int trail; - unsigned int top; - /* - unpack_stack* stack; - unsigned int stack_size; - unpack_stack embed_stack[MSGPACK_EMBED_STACK_SIZE]; - */ - unpack_stack stack[MSGPACK_EMBED_STACK_SIZE]; + unpack_user user; + unsigned int cs; + unsigned int trail; + unsigned int top; + /* + unpack_stack* stack; + unsigned int stack_size; + unpack_stack embed_stack[MSGPACK_EMBED_STACK_SIZE]; + */ + unpack_stack stack[MSGPACK_EMBED_STACK_SIZE]; }; static inline void unpack_init(unpack_context* ctx) { - ctx->cs = CS_HEADER; - ctx->trail = 0; - ctx->top = 0; - /* - ctx->stack = ctx->embed_stack; - ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; - */ - ctx->stack[0].obj = unpack_callback_root(&ctx->user); + ctx->cs = CS_HEADER; + ctx->trail = 0; + ctx->top = 0; + /* + ctx->stack = ctx->embed_stack; + ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; + */ + ctx->stack[0].obj = unpack_callback_root(&ctx->user); } /* static inline void unpack_destroy(unpack_context* ctx) { - if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { - free(ctx->stack); - } + if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { + free(ctx->stack); + } } */ static inline PyObject* unpack_data(unpack_context* ctx) { - return (ctx)->stack[0].obj; + return (ctx)->stack[0].obj; } template static inline int unpack_execute(unpack_context* ctx, const char* data, size_t len, size_t* off) { - assert(len >= *off); + assert(len >= *off); - const unsigned char* p = (unsigned char*)data + *off; - const unsigned char* const pe = (unsigned char*)data + len; - const void* n = NULL; + const unsigned char* p = (unsigned char*)data + *off; + const unsigned char* const pe = (unsigned char*)data + len; + const void* n = NULL; - unsigned int trail = ctx->trail; - unsigned int cs = ctx->cs; - unsigned int top = ctx->top; - unpack_stack* stack = ctx->stack; - /* - unsigned int stack_size = ctx->stack_size; - */ - unpack_user* user = &ctx->user; + unsigned int trail = ctx->trail; + unsigned int cs = ctx->cs; + unsigned int top = ctx->top; + unpack_stack* stack = ctx->stack; + /* + unsigned int stack_size = ctx->stack_size; + */ + unpack_user* user = &ctx->user; - PyObject* obj; - unpack_stack* c = NULL; + PyObject* obj; + unpack_stack* c = NULL; - int ret; + int ret; #define construct_cb(name) \ construct && unpack_callback ## name #define push_simple_value(func) \ - if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \ - goto _push + if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \ + goto _push #define push_fixed_value(func, arg) \ - if(construct_cb(func)(user, arg, &obj) < 0) { goto _failed; } \ - goto _push + if(construct_cb(func)(user, arg, &obj) < 0) { goto _failed; } \ + goto _push #define push_variable_value(func, base, pos, len) \ - if(construct_cb(func)(user, \ - (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ - goto _push + if(construct_cb(func)(user, \ + (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ + goto _push #define again_fixed_trail(_cs, trail_len) \ - trail = trail_len; \ - cs = _cs; \ - goto _fixed_trail_again + trail = trail_len; \ + cs = _cs; \ + goto _fixed_trail_again #define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ - trail = trail_len; \ - if(trail == 0) { goto ifzero; } \ - cs = _cs; \ - goto _fixed_trail_again + trail = trail_len; \ + if(trail == 0) { goto ifzero; } \ + cs = _cs; \ + goto _fixed_trail_again #define start_container(func, count_, ct_) \ - if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \ - if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ - if((count_) == 0) { obj = stack[top].obj; \ - if (construct_cb(func##_end)(user, &obj) < 0) { goto _failed; } \ - goto _push; } \ - stack[top].ct = ct_; \ - stack[top].size = count_; \ - stack[top].count = 0; \ - ++top; \ - /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ - /*printf("stack push %d\n", top);*/ \ - /* FIXME \ - if(top >= stack_size) { \ - if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \ - size_t csize = sizeof(unpack_stack) * MSGPACK_EMBED_STACK_SIZE; \ - size_t nsize = csize * 2; \ - unpack_stack* tmp = (unpack_stack*)malloc(nsize); \ - if(tmp == NULL) { goto _failed; } \ - memcpy(tmp, ctx->stack, csize); \ - ctx->stack = stack = tmp; \ - ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ - } else { \ - size_t nsize = sizeof(unpack_stack) * ctx->stack_size * 2; \ - unpack_stack* tmp = (unpack_stack*)realloc(ctx->stack, nsize); \ - if(tmp == NULL) { goto _failed; } \ - ctx->stack = stack = tmp; \ - ctx->stack_size = stack_size = stack_size * 2; \ - } \ - } \ - */ \ - goto _header_again + if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \ + if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ + if((count_) == 0) { obj = stack[top].obj; \ + if (construct_cb(func##_end)(user, &obj) < 0) { goto _failed; } \ + goto _push; } \ + stack[top].ct = ct_; \ + stack[top].size = count_; \ + stack[top].count = 0; \ + ++top; \ + /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ + /*printf("stack push %d\n", top);*/ \ + /* FIXME \ + if(top >= stack_size) { \ + if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \ + size_t csize = sizeof(unpack_stack) * MSGPACK_EMBED_STACK_SIZE; \ + size_t nsize = csize * 2; \ + unpack_stack* tmp = (unpack_stack*)malloc(nsize); \ + if(tmp == NULL) { goto _failed; } \ + memcpy(tmp, ctx->stack, csize); \ + ctx->stack = stack = tmp; \ + ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ + } else { \ + size_t nsize = sizeof(unpack_stack) * ctx->stack_size * 2; \ + unpack_stack* tmp = (unpack_stack*)realloc(ctx->stack, nsize); \ + if(tmp == NULL) { goto _failed; } \ + ctx->stack = stack = tmp; \ + ctx->stack_size = stack_size = stack_size * 2; \ + } \ + } \ + */ \ + goto _header_again #define NEXT_CS(p) \ - ((unsigned int)*p & 0x1f) + ((unsigned int)*p & 0x1f) #ifdef USE_CASE_RANGE #define SWITCH_RANGE_BEGIN switch(*p) { @@ -166,221 +166,221 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l #define SWITCH_RANGE_END } } #endif - if(p == pe) { goto _out; } - do { - switch(cs) { - case CS_HEADER: - SWITCH_RANGE_BEGIN - SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum - push_fixed_value(_uint8, *(uint8_t*)p); - SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum - push_fixed_value(_int8, *(int8_t*)p); - SWITCH_RANGE(0xc0, 0xdf) // Variable - switch(*p) { - case 0xc0: // nil - push_simple_value(_nil); - //case 0xc1: // string - // again_terminal_trail(NEXT_CS(p), p+1); - case 0xc2: // false - push_simple_value(_false); - case 0xc3: // true - push_simple_value(_true); - //case 0xc4: - //case 0xc5: - //case 0xc6: - //case 0xc7: - //case 0xc8: - //case 0xc9: - case 0xca: // float - case 0xcb: // double - case 0xcc: // unsigned int 8 - case 0xcd: // unsigned int 16 - case 0xce: // unsigned int 32 - case 0xcf: // unsigned int 64 - case 0xd0: // signed int 8 - case 0xd1: // signed int 16 - case 0xd2: // signed int 32 - case 0xd3: // signed int 64 - again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); - //case 0xd4: - //case 0xd5: - //case 0xd6: // big integer 16 - //case 0xd7: // big integer 32 - //case 0xd8: // big float 16 - //case 0xd9: // big float 32 - case 0xda: // raw 16 - case 0xdb: // raw 32 - case 0xdc: // array 16 - case 0xdd: // array 32 - case 0xde: // map 16 - case 0xdf: // map 32 - again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); - default: - goto _failed; - } - SWITCH_RANGE(0xa0, 0xbf) // FixRaw - again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); - SWITCH_RANGE(0x90, 0x9f) // FixArray - start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); - SWITCH_RANGE(0x80, 0x8f) // FixMap - start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); + if(p == pe) { goto _out; } + do { + switch(cs) { + case CS_HEADER: + SWITCH_RANGE_BEGIN + SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum + push_fixed_value(_uint8, *(uint8_t*)p); + SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum + push_fixed_value(_int8, *(int8_t*)p); + SWITCH_RANGE(0xc0, 0xdf) // Variable + switch(*p) { + case 0xc0: // nil + push_simple_value(_nil); + //case 0xc1: // string + // again_terminal_trail(NEXT_CS(p), p+1); + case 0xc2: // false + push_simple_value(_false); + case 0xc3: // true + push_simple_value(_true); + //case 0xc4: + //case 0xc5: + //case 0xc6: + //case 0xc7: + //case 0xc8: + //case 0xc9: + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); + //case 0xd4: + //case 0xd5: + //case 0xd6: // big integer 16 + //case 0xd7: // big integer 32 + //case 0xd8: // big float 16 + //case 0xd9: // big float 32 + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); + default: + goto _failed; + } + SWITCH_RANGE(0xa0, 0xbf) // FixRaw + again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); + SWITCH_RANGE(0x90, 0x9f) // FixArray + start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); + SWITCH_RANGE(0x80, 0x8f) // FixMap + start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); - SWITCH_RANGE_DEFAULT - goto _failed; - SWITCH_RANGE_END - // end CS_HEADER + SWITCH_RANGE_DEFAULT + goto _failed; + SWITCH_RANGE_END + // end CS_HEADER - _fixed_trail_again: - ++p; + _fixed_trail_again: + ++p; - default: - if((size_t)(pe - p) < trail) { goto _out; } - n = p; p += trail - 1; - switch(cs) { - //case CS_ - //case CS_ - case CS_FLOAT: { - union { uint32_t i; float f; } mem; - mem.i = _msgpack_load32(uint32_t,n); - push_fixed_value(_float, mem.f); } - case CS_DOUBLE: { - union { uint64_t i; double f; } mem; - mem.i = _msgpack_load64(uint64_t,n); + default: + if((size_t)(pe - p) < trail) { goto _out; } + n = p; p += trail - 1; + switch(cs) { + //case CS_ + //case CS_ + case CS_FLOAT: { + union { uint32_t i; float f; } mem; + mem.i = _msgpack_load32(uint32_t,n); + push_fixed_value(_float, mem.f); } + case CS_DOUBLE: { + union { uint64_t i; double f; } mem; + mem.i = _msgpack_load64(uint64_t,n); #if defined(__arm__) && !(__ARM_EABI__) // arm-oabi // https://github.com/msgpack/msgpack-perl/pull/1 mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL); #endif - push_fixed_value(_double, mem.f); } - case CS_UINT_8: - push_fixed_value(_uint8, *(uint8_t*)n); - case CS_UINT_16: - push_fixed_value(_uint16, _msgpack_load16(uint16_t,n)); - case CS_UINT_32: - push_fixed_value(_uint32, _msgpack_load32(uint32_t,n)); - case CS_UINT_64: - push_fixed_value(_uint64, _msgpack_load64(uint64_t,n)); + push_fixed_value(_double, mem.f); } + case CS_UINT_8: + push_fixed_value(_uint8, *(uint8_t*)n); + case CS_UINT_16: + push_fixed_value(_uint16, _msgpack_load16(uint16_t,n)); + case CS_UINT_32: + push_fixed_value(_uint32, _msgpack_load32(uint32_t,n)); + case CS_UINT_64: + push_fixed_value(_uint64, _msgpack_load64(uint64_t,n)); - case CS_INT_8: - push_fixed_value(_int8, *(int8_t*)n); - case CS_INT_16: - push_fixed_value(_int16, _msgpack_load16(int16_t,n)); - case CS_INT_32: - push_fixed_value(_int32, _msgpack_load32(int32_t,n)); - case CS_INT_64: - push_fixed_value(_int64, _msgpack_load64(int64_t,n)); + case CS_INT_8: + push_fixed_value(_int8, *(int8_t*)n); + case CS_INT_16: + push_fixed_value(_int16, _msgpack_load16(int16_t,n)); + case CS_INT_32: + push_fixed_value(_int32, _msgpack_load32(int32_t,n)); + case CS_INT_64: + push_fixed_value(_int64, _msgpack_load64(int64_t,n)); - //case CS_ - //case CS_ - //case CS_BIG_INT_16: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero); - //case CS_BIG_INT_32: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero); - //case ACS_BIG_INT_VALUE: - //_big_int_zero: - // // FIXME - // push_variable_value(_big_int, data, n, trail); + //case CS_ + //case CS_ + //case CS_BIG_INT_16: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero); + //case CS_BIG_INT_32: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero); + //case ACS_BIG_INT_VALUE: + //_big_int_zero: + // // FIXME + // push_variable_value(_big_int, data, n, trail); - //case CS_BIG_FLOAT_16: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero); - //case CS_BIG_FLOAT_32: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero); - //case ACS_BIG_FLOAT_VALUE: - //_big_float_zero: - // // FIXME - // push_variable_value(_big_float, data, n, trail); + //case CS_BIG_FLOAT_16: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero); + //case CS_BIG_FLOAT_32: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero); + //case ACS_BIG_FLOAT_VALUE: + //_big_float_zero: + // // FIXME + // push_variable_value(_big_float, data, n, trail); - case CS_RAW_16: - again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero); - case CS_RAW_32: - again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero); - case ACS_RAW_VALUE: - _raw_zero: - push_variable_value(_raw, data, n, trail); + case CS_RAW_16: + again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero); + case CS_RAW_32: + again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero); + case ACS_RAW_VALUE: + _raw_zero: + push_variable_value(_raw, data, n, trail); - case CS_ARRAY_16: - start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM); - case CS_ARRAY_32: - /* FIXME security guard */ - start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM); + case CS_ARRAY_16: + start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM); + case CS_ARRAY_32: + /* FIXME security guard */ + start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM); - case CS_MAP_16: - start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY); - case CS_MAP_32: - /* FIXME security guard */ - start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); + case CS_MAP_16: + start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY); + case CS_MAP_32: + /* FIXME security guard */ + start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); - default: - goto _failed; - } - } + default: + goto _failed; + } + } _push: - if(top == 0) { goto _finish; } - c = &stack[top-1]; - switch(c->ct) { - case CT_ARRAY_ITEM: - if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } - if(++c->count == c->size) { - obj = c->obj; - if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; } - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - goto _header_again; - case CT_MAP_KEY: - c->map_key = obj; - c->ct = CT_MAP_VALUE; - goto _header_again; - case CT_MAP_VALUE: - if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } - if(++c->count == c->size) { - obj = c->obj; - if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; } - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - c->ct = CT_MAP_KEY; - goto _header_again; + if(top == 0) { goto _finish; } + c = &stack[top-1]; + switch(c->ct) { + case CT_ARRAY_ITEM: + if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; } + if(++c->count == c->size) { + obj = c->obj; + if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; } + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + goto _header_again; + case CT_MAP_KEY: + c->map_key = obj; + c->ct = CT_MAP_VALUE; + goto _header_again; + case CT_MAP_VALUE: + if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } + if(++c->count == c->size) { + obj = c->obj; + if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; } + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + c->ct = CT_MAP_KEY; + goto _header_again; - default: - goto _failed; - } + default: + goto _failed; + } _header_again: - cs = CS_HEADER; - ++p; - } while(p != pe); - goto _out; + cs = CS_HEADER; + ++p; + } while(p != pe); + goto _out; _finish: - if (!construct) - unpack_callback_nil(user, &obj); - stack[0].obj = obj; - ++p; - ret = 1; - /*printf("-- finish --\n"); */ - goto _end; + if (!construct) + unpack_callback_nil(user, &obj); + stack[0].obj = obj; + ++p; + ret = 1; + /*printf("-- finish --\n"); */ + goto _end; _failed: - /*printf("** FAILED **\n"); */ - ret = -1; - goto _end; + /*printf("** FAILED **\n"); */ + ret = -1; + goto _end; _out: - ret = 0; - goto _end; + ret = 0; + goto _end; _end: - ctx->cs = cs; - ctx->trail = trail; - ctx->top = top; - *off = p - (const unsigned char*)data; + ctx->cs = cs; + ctx->trail = trail; + ctx->top = top; + *off = p - (const unsigned char*)data; - return ret; + return ret; #undef construct_cb } @@ -398,53 +398,53 @@ _end: template static inline int unpack_container_header(unpack_context* ctx, const char* data, size_t len, size_t* off) { - assert(len >= *off); - uint32_t size; - const unsigned char *const p = (unsigned char*)data + *off; + assert(len >= *off); + uint32_t size; + const unsigned char *const p = (unsigned char*)data + *off; #define inc_offset(inc) \ - if (len - *off < inc) \ - return 0; \ - *off += inc; + if (len - *off < inc) \ + return 0; \ + *off += inc; - switch (*p) { - case var_offset: - inc_offset(3); - size = _msgpack_load16(uint16_t, p + 1); - break; - case var_offset + 1: - inc_offset(5); - size = _msgpack_load32(uint32_t, p + 1); - break; + switch (*p) { + case var_offset: + inc_offset(3); + size = _msgpack_load16(uint16_t, p + 1); + break; + case var_offset + 1: + inc_offset(5); + size = _msgpack_load32(uint32_t, p + 1); + break; #ifdef USE_CASE_RANGE - case fixed_offset + 0x0 ... fixed_offset + 0xf: + case fixed_offset + 0x0 ... fixed_offset + 0xf: #else - case fixed_offset + 0x0: - case fixed_offset + 0x1: - case fixed_offset + 0x2: - case fixed_offset + 0x3: - case fixed_offset + 0x4: - case fixed_offset + 0x5: - case fixed_offset + 0x6: - case fixed_offset + 0x7: - case fixed_offset + 0x8: - case fixed_offset + 0x9: - case fixed_offset + 0xa: - case fixed_offset + 0xb: - case fixed_offset + 0xc: - case fixed_offset + 0xd: - case fixed_offset + 0xe: - case fixed_offset + 0xf: + case fixed_offset + 0x0: + case fixed_offset + 0x1: + case fixed_offset + 0x2: + case fixed_offset + 0x3: + case fixed_offset + 0x4: + case fixed_offset + 0x5: + case fixed_offset + 0x6: + case fixed_offset + 0x7: + case fixed_offset + 0x8: + case fixed_offset + 0x9: + case fixed_offset + 0xa: + case fixed_offset + 0xb: + case fixed_offset + 0xc: + case fixed_offset + 0xd: + case fixed_offset + 0xe: + case fixed_offset + 0xf: #endif - ++*off; - size = ((unsigned int)*p) & 0x0f; - break; - default: - PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); - return -1; + ++*off; + size = ((unsigned int)*p) & 0x0f; + break; + default: + PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); + return -1; } - unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj); - return 1; + unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj); + return 1; } #undef SWITCH_RANGE_BEGIN @@ -459,4 +459,4 @@ static const execute_fn read_map_header = &unpack_container_header<0x80, 0xde>; #undef NEXT_CS -/* vim: set ts=4 sw=4 noexpandtab */ +/* vim: set ts=4 sw=4 sts=4 expandtab */ From 5c90f953dae37610f478183849b77413d6ec8e9f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 13 Mar 2013 18:15:43 +0900 Subject: [PATCH 0696/1172] Add build batch for Windows --- build_windows.bat | 24 ++++++++++++++++++++++++ upload_windows.bat | 4 ++++ 2 files changed, 28 insertions(+) create mode 100644 build_windows.bat create mode 100644 upload_windows.bat diff --git a/build_windows.bat b/build_windows.bat new file mode 100644 index 0000000..a71c0e0 --- /dev/null +++ b/build_windows.bat @@ -0,0 +1,24 @@ +set MSSdk=1 +set DISTUTILS_USE_SDK=1 + +rem Python27 x86 +rem call "C:\Program Files\Microsoft SDKs\Windows\v6.1\Bin\SetEnv.cmd" /Release /x86 /xp +call "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars32.bat" +c:\Python27\python setup.py build_ext -f build install +pause + +rem Python27 amd64 +rem call "C:\Program Files\Microsoft SDKs\Windows\v6.1\Bin\SetEnv.cmd" /Release /x64 /xp +call "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars64.bat" +c:\Python27_amd64\python setup.py build_ext -f build install +pause + +rem Python33 x86 +call "C:\Program Files\Microsoft SDKs\Windows\v7.1\bin\SetEnv.cmd" /Release /x86 /xp +c:\Python33\python setup.py build_ext -f build install +pause + +rem Python33 amd64 +call "C:\Program Files\Microsoft SDKs\Windows\v7.1\bin\SetEnv.cmd" /Release /x64 /xp +c:\Python33_amd64\python setup.py build_ext -f build install +pause diff --git a/upload_windows.bat b/upload_windows.bat new file mode 100644 index 0000000..5cd9a7c --- /dev/null +++ b/upload_windows.bat @@ -0,0 +1,4 @@ +c:\Python27\python setup.py bdist_egg bdist_wininst upload +c:\Python33\python setup.py bdist_egg bdist_wininst upload +c:\Python27_amd64\python setup.py bdist_egg bdist_wininst upload +c:\Python33_amd64\python setup.py bdist_egg bdist_wininst upload From 0faa1bb558ab671b9503c5e22a97c0ec24ab5a19 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 8 Apr 2013 01:57:37 +0900 Subject: [PATCH 0697/1172] Remove unnecessary type declaration. --- msgpack/_packer.pyx | 6 ------ msgpack/_unpacker.pyx | 3 +-- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 562c92c..93323c4 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -2,12 +2,6 @@ #cython: embedsignature=True from cpython cimport * -cdef extern from "Python.h": - ctypedef char* const_char_ptr "const char*" - ctypedef char* const_void_ptr "const void*" - ctypedef struct PyObject - cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1 - from libc.stdlib cimport * from libc.string cimport * from libc.limits cimport * diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 5813dc9..1ad71a0 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -3,7 +3,6 @@ from cpython cimport * cdef extern from "Python.h": - ctypedef char* const_char_ptr "const char*" ctypedef char* const_void_ptr "const void*" ctypedef struct PyObject cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1 @@ -37,7 +36,7 @@ cdef extern from "unpack.h": unsigned int ct PyObject* key - ctypedef int (*execute_fn)(unpack_context* ctx, const_char_ptr data, + ctypedef int (*execute_fn)(unpack_context* ctx, const char* data, size_t len, size_t* off) except? -1 execute_fn unpack_construct execute_fn unpack_skip From 18215b01bb272f28d98af5c9373fac8cbb24b30e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 8 Apr 2013 10:52:11 +0900 Subject: [PATCH 0698/1172] Unpacker.feed() uses new buffer interface. --- msgpack/_unpacker.pyx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 1ad71a0..2644f15 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -254,13 +254,15 @@ cdef class Unpacker(object): def feed(self, object next_bytes): """Append `next_bytes` to internal buffer.""" - cdef char* buf - cdef Py_ssize_t buf_len + cdef Py_buffer pybuff if self.file_like is not None: raise AssertionError( "unpacker.feed() is not be able to use with `file_like`.") - PyObject_AsReadBuffer(next_bytes, &buf, &buf_len) - self.append_buffer(buf, buf_len) + PyObject_GetBuffer(next_bytes, &pybuff, PyBUF_SIMPLE) + try: + self.append_buffer(pybuff.buf, pybuff.len) + finally: + PyBuffer_Release(&pybuff) cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len): cdef: From 085db7f8dca2b4e2497bfb291238cd3ff2d06e28 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 8 Apr 2013 10:57:21 +0900 Subject: [PATCH 0699/1172] Use new buffer interface. --- msgpack/_unpacker.pyx | 49 ++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 2644f15..46be7e0 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -2,10 +2,6 @@ #cython: embedsignature=True from cpython cimport * -cdef extern from "Python.h": - ctypedef char* const_void_ptr "const void*" - ctypedef struct PyObject - cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1 from libc.stdlib cimport * from libc.string cimport * @@ -19,8 +15,8 @@ from msgpack.exceptions import ( ) - cdef extern from "unpack.h": + ctypedef struct PyObject ctypedef struct msgpack_user: bint use_list PyObject* object_hook @@ -91,32 +87,33 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef size_t off = 0 cdef int ret - cdef char* buf - cdef Py_ssize_t buf_len + cdef Py_buffer buff cdef char* cenc = NULL cdef char* cerr = NULL - PyObject_AsReadBuffer(packed, &buf, &buf_len) + PyObject_GetBuffer(packed, &buff, PyBUF_SIMPLE) + try: + if encoding is not None: + if isinstance(encoding, unicode): + encoding = encoding.encode('ascii') + cenc = PyBytes_AsString(encoding) - if encoding is not None: - if isinstance(encoding, unicode): - encoding = encoding.encode('ascii') - cenc = PyBytes_AsString(encoding) + if unicode_errors is not None: + if isinstance(unicode_errors, unicode): + unicode_errors = unicode_errors.encode('ascii') + cerr = PyBytes_AsString(unicode_errors) - if unicode_errors is not None: - if isinstance(unicode_errors, unicode): - unicode_errors = unicode_errors.encode('ascii') - cerr = PyBytes_AsString(unicode_errors) - - init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) - ret = unpack_construct(&ctx, buf, buf_len, &off) - if ret == 1: - obj = unpack_data(&ctx) - if off < buf_len: - raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) - return obj - else: - raise UnpackValueError("Unpack failed: error = %d" % (ret,)) + init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) + ret = unpack_construct(&ctx, buff.buf, buff.len, &off) + if ret == 1: + obj = unpack_data(&ctx) + if off < buff.len: + raise ExtraData(obj, PyBytes_FromStringAndSize((buff)+off, buff.len-off)) + return obj + else: + raise UnpackValueError("Unpack failed: error = %d" % (ret,)) + finally: + PyBuffer_Release(&buff) def unpack(object stream, object object_hook=None, object list_hook=None, From a692bf98520b7836f6b456c1fcc33f20bb45f952 Mon Sep 17 00:00:00 2001 From: jnothman Date: Wed, 10 Apr 2013 21:03:41 +1000 Subject: [PATCH 0700/1172] Remove obsolete StopIteration warning --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index da2f023..294cd63 100644 --- a/README.rst +++ b/README.rst @@ -151,9 +151,6 @@ the result, or ignoring it. The latter two methods return the number of elements in the upcoming container, so that each element in an array, or key-value pair in a map, can be unpacked or skipped individually. -Warning: these methods raise ``StopIteration`` when called at the end of the -stream. Unless caught, this may silently break an iteration. - Each of these methods may optionally write the packed data it reads to a callback function: From 0c3fecf91bb39c6672fbf2f56d1c6e9a25ff90f0 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Wed, 1 May 2013 12:15:12 +0900 Subject: [PATCH 0701/1172] fix a typo in a comment --- msgpack/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index f9a2f5e..c66592c 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -147,7 +147,7 @@ class Unpacker(object): example of streaming deserialization from socket:: - unpacker = Unapcker() + unpacker = Unpacker() while 1: buf = sock.recv(1024*2) if not buf: From 56dbf7f9beba6a01040f7a513f85a8079f7b26f3 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Tue, 7 May 2013 13:56:39 +0900 Subject: [PATCH 0702/1172] fix more comment typos --- msgpack/_packer.pyx | 2 +- msgpack/fallback.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 93323c4..6289192 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -49,7 +49,7 @@ cdef class Packer(object): See also simplejson's document. :param str encoding: Convert unicode to bytes with this encoding. (default: 'utf-8') - :param str unicode_erros: + :param str unicode_errors: Error handler for encoding unicode. (default: 'strict') :param bool use_single_float: Use single precision float type for float. (default: False) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index c66592c..8f9d646 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -439,7 +439,7 @@ class Packer(object): See also simplejson's document. :param str encoding: Convert unicode to bytes with this encoding. (default: 'utf-8') - :param str unicode_erros: + :param str unicode_errors: Error handler for encoding unicode. (default: 'strict') :param bool use_single_float: Use single precision float type for float. (default: False) From b0c193f3e07fdabc3d489bd5e134e3e41733f204 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Thu, 16 May 2013 12:41:02 +0900 Subject: [PATCH 0703/1172] fix long/int confusions in pyx version of unpack --- msgpack/unpack.h | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 595b8df..fb13b4e 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -68,7 +68,12 @@ static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unp static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) { - PyObject *p = PyLong_FromUnsignedLongLong(d); + PyObject *p; + if (d > LONG_MAX) { + p = PyLong_FromUnsignedLongLong((unsigned long)d); + } else { + p = PyInt_FromLong((long)d); + } if (!p) return -1; *o = p; @@ -96,9 +101,12 @@ static inline int unpack_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_ static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) { - PyObject *p = PyLong_FromLongLong(d); - if (!p) - return -1; + PyObject *p; + if (d > LONG_MAX || d < LONG_MIN) { + p = PyLong_FromLongLong((unsigned long)d); + } else { + p = PyInt_FromLong((long)d); + } *o = p; return 0; } From 63b9fa5843ad9e38e3797c061de495f49659ab08 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Thu, 16 May 2013 12:57:28 +0900 Subject: [PATCH 0704/1172] fix a compilation error msgpack/_unpacker.pyx: In function 'PyObject* __pyx_pf_7msgpack_9_unpacker_unpac kb(PyObject*, PyObject*, PyObject*, PyObject*, int, PyObject*, PyObject*, PyObje ct*)': msgpack/_unpacker.pyx:111:70: error: invalid cast from type 'Py_buffer' to type 'char*' --- msgpack/_unpacker.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 46be7e0..06806bd 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -108,7 +108,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, if ret == 1: obj = unpack_data(&ctx) if off < buff.len: - raise ExtraData(obj, PyBytes_FromStringAndSize((buff)+off, buff.len-off)) + raise ExtraData(obj, PyBytes_FromStringAndSize(buff.buf+off, buff.len-off)) return obj else: raise UnpackValueError("Unpack failed: error = %d" % (ret,)) From 08c56d66f6eecb99850897dd3c4987dea01e6d44 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 19 May 2013 01:13:21 +0900 Subject: [PATCH 0705/1172] Use --cplus for cythoning --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3fe278e..71be6af 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ doc: cd docs && make zip cython: - cython msgpack/*.pyx + cython --cplus msgpack/*.pyx python3: cython python3 setup.py build_ext -i -f From bbe86e7a925f113337f6e24d3ff24ddc30137dd6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 19 May 2013 12:30:23 +0900 Subject: [PATCH 0706/1172] Revert "Use new buffer interface." This reverts commit 085db7f8dca2b4e2497bfb291238cd3ff2d06e28. Conflicts: msgpack/_unpacker.pyx --- msgpack/_unpacker.pyx | 49 +++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 06806bd..2644f15 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -2,6 +2,10 @@ #cython: embedsignature=True from cpython cimport * +cdef extern from "Python.h": + ctypedef char* const_void_ptr "const void*" + ctypedef struct PyObject + cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1 from libc.stdlib cimport * from libc.string cimport * @@ -15,8 +19,8 @@ from msgpack.exceptions import ( ) + cdef extern from "unpack.h": - ctypedef struct PyObject ctypedef struct msgpack_user: bint use_list PyObject* object_hook @@ -87,33 +91,32 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef size_t off = 0 cdef int ret - cdef Py_buffer buff + cdef char* buf + cdef Py_ssize_t buf_len cdef char* cenc = NULL cdef char* cerr = NULL - PyObject_GetBuffer(packed, &buff, PyBUF_SIMPLE) - try: - if encoding is not None: - if isinstance(encoding, unicode): - encoding = encoding.encode('ascii') - cenc = PyBytes_AsString(encoding) + PyObject_AsReadBuffer(packed, &buf, &buf_len) - if unicode_errors is not None: - if isinstance(unicode_errors, unicode): - unicode_errors = unicode_errors.encode('ascii') - cerr = PyBytes_AsString(unicode_errors) + if encoding is not None: + if isinstance(encoding, unicode): + encoding = encoding.encode('ascii') + cenc = PyBytes_AsString(encoding) - init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) - ret = unpack_construct(&ctx, buff.buf, buff.len, &off) - if ret == 1: - obj = unpack_data(&ctx) - if off < buff.len: - raise ExtraData(obj, PyBytes_FromStringAndSize(buff.buf+off, buff.len-off)) - return obj - else: - raise UnpackValueError("Unpack failed: error = %d" % (ret,)) - finally: - PyBuffer_Release(&buff) + if unicode_errors is not None: + if isinstance(unicode_errors, unicode): + unicode_errors = unicode_errors.encode('ascii') + cerr = PyBytes_AsString(unicode_errors) + + init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) + ret = unpack_construct(&ctx, buf, buf_len, &off) + if ret == 1: + obj = unpack_data(&ctx) + if off < buf_len: + raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) + return obj + else: + raise UnpackValueError("Unpack failed: error = %d" % (ret,)) def unpack(object stream, object object_hook=None, object list_hook=None, From 956f55ecdf3b98e8072b4073da650e9a8fa05e0b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 19 May 2013 12:32:33 +0900 Subject: [PATCH 0707/1172] Stop using const_void_ptr typedef. New Cython supports const natively. --- msgpack/_unpacker.pyx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 2644f15..9ff1085 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -3,9 +3,8 @@ from cpython cimport * cdef extern from "Python.h": - ctypedef char* const_void_ptr "const void*" ctypedef struct PyObject - cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1 + cdef int PyObject_AsReadBuffer(object o, const void* buff, Py_ssize_t* buf_len) except -1 from libc.stdlib cimport * from libc.string cimport * @@ -96,7 +95,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef char* cenc = NULL cdef char* cerr = NULL - PyObject_AsReadBuffer(packed, &buf, &buf_len) + PyObject_AsReadBuffer(packed, &buf, &buf_len) if encoding is not None: if isinstance(encoding, unicode): From 3dbb2d1e7e97a61a9c36e316336552dc0e20577f Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Mon, 3 Jun 2013 13:18:52 +0900 Subject: [PATCH 0708/1172] fix compilation errors --- msgpack/_unpacker.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 9ff1085..1f4dd85 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -4,7 +4,7 @@ from cpython cimport * cdef extern from "Python.h": ctypedef struct PyObject - cdef int PyObject_AsReadBuffer(object o, const void* buff, Py_ssize_t* buf_len) except -1 + cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1 from libc.stdlib cimport * from libc.string cimport * @@ -95,7 +95,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef char* cenc = NULL cdef char* cerr = NULL - PyObject_AsReadBuffer(packed, &buf, &buf_len) + PyObject_AsReadBuffer(packed, &buf, &buf_len) if encoding is not None: if isinstance(encoding, unicode): From d1b9ecbc8e33663663f2fb2f13ac17854dd6c8d6 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Mon, 3 Jun 2013 13:49:44 +0900 Subject: [PATCH 0709/1172] fix long vs long long bugs these bugs were introduced by "fix long/int confusions in pyx version of unpack" commit. --- msgpack/unpack.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index fb13b4e..baeed1f 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -70,7 +70,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp { PyObject *p; if (d > LONG_MAX) { - p = PyLong_FromUnsignedLongLong((unsigned long)d); + p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d); } else { p = PyInt_FromLong((long)d); } @@ -103,7 +103,7 @@ static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpac { PyObject *p; if (d > LONG_MAX || d < LONG_MIN) { - p = PyLong_FromLongLong((unsigned long)d); + p = PyLong_FromLongLong((unsigned PY_LONG_LONG)d); } else { p = PyInt_FromLong((long)d); } From e250b89920527d69e30566746a1ef12a2f43b25c Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Mon, 3 Jun 2013 13:17:03 +0900 Subject: [PATCH 0710/1172] more tests --- test/test_pack.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test_pack.py b/test/test_pack.py index 3225f41..9dc5ada 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -12,11 +12,12 @@ from io import BytesIO def check(data, use_list=False): re = unpackb(packb(data), use_list=use_list) assert re == data + assert type(re) == type(data) def testPack(): test_data = [ - 0, 1, 127, 128, 255, 256, 65535, 65536, - -1, -32, -33, -128, -129, -32768, -32769, + 0, 1, 127, 128, 255, 256, 65535, 65536, 4294967295, 4294967296, + -1, -32, -33, -128, -129, -32768, -32769, -4294967296, -4294967297, 1.0, b"", b"a", b"a"*31, b"a"*32, None, True, False, From 12f87147b5e2c1ffadc9fed51dca35d940038798 Mon Sep 17 00:00:00 2001 From: Lieven Govaerts Date: Fri, 13 Sep 2013 13:47:13 +0200 Subject: [PATCH 0711/1172] * msgpack/exceptions.py: Fix typo in error message. --- msgpack/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py index 2565541..f7678f1 100644 --- a/msgpack/exceptions.py +++ b/msgpack/exceptions.py @@ -20,7 +20,7 @@ class ExtraData(ValueError): self.extra = extra def __str__(self): - return "unpack(b) recieved extra data." + return "unpack(b) received extra data." class PackException(Exception): pass From d61097511a1caa0e3bc5a70c1d2d92f448bd5025 Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Tue, 15 Oct 2013 16:59:43 +0200 Subject: [PATCH 0712/1172] add support for extended types: you can now pack/unpack custom python objects by subclassing Packer and Unpacker --- msgpack/fallback.py | 131 ++++++++++++++++++++++++++--------------- test/test_extension.py | 24 ++++++++ 2 files changed, 108 insertions(+), 47 deletions(-) create mode 100644 test/test_extension.py diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 8f9d646..b7f455b 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -58,55 +58,10 @@ TYPE_ARRAY = 1 TYPE_MAP = 2 TYPE_RAW = 3 +EXTENDED_TYPE = 1000 + DEFAULT_RECURSE_LIMIT=511 -def pack(o, stream, **kwargs): - """ - Pack object `o` and write it to `stream` - - See :class:`Packer` for options. - """ - packer = Packer(**kwargs) - stream.write(packer.pack(o)) - -def packb(o, **kwargs): - """ - Pack object `o` and return packed bytes - - See :class:`Packer` for options. - """ - return Packer(**kwargs).pack(o) - -def unpack(stream, **kwargs): - """ - Unpack an object from `stream`. - - Raises `ExtraData` when `packed` contains extra bytes. - See :class:`Unpacker` for options. - """ - unpacker = Unpacker(stream, **kwargs) - ret = unpacker._fb_unpack() - if unpacker._fb_got_extradata(): - raise ExtraData(ret, unpacker._fb_get_extradata()) - return ret - -def unpackb(packed, **kwargs): - """ - Unpack an object from `packed`. - - Raises `ExtraData` when `packed` contains extra bytes. - See :class:`Unpacker` for options. - """ - unpacker = Unpacker(None, **kwargs) - unpacker.feed(packed) - try: - ret = unpacker._fb_unpack() - except OutOfData: - raise UnpackValueError("Data is not enough.") - if unpacker._fb_got_extradata(): - raise ExtraData(ret, unpacker._fb_get_extradata()) - return ret - class Unpacker(object): """ Streaming unpacker. @@ -334,6 +289,9 @@ class Unpacker(object): elif b == 0xdf: n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] typ = TYPE_MAP + elif b == 0xc9: + n, typ = struct.unpack(">Ib", self._fb_read(5, write_bytes)) + typ += EXTENDED_TYPE else: raise UnpackValueError("Unknown header: 0x%x" % b) return typ, n, obj @@ -390,6 +348,10 @@ class Unpacker(object): if self._encoding is not None: obj = obj.decode(self._encoding, self._unicode_errors) return obj + if typ >= EXTENDED_TYPE: + typ -= EXTENDED_TYPE + data = self._fb_read(n, write_bytes) + return self.handle_extended_type(typ, data) assert typ == TYPE_IMMEDIATE return obj @@ -411,6 +373,9 @@ class Unpacker(object): self._fb_consume() return ret + def handle_extended_type(self, typecode, data): + raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode) + def read_array_header(self, write_bytes=None): ret = self._fb_unpack(EX_READ_ARRAY_HEADER, write_bytes) self._fb_consume() @@ -521,10 +486,33 @@ class Packer(object): if isinstance(obj, dict): return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj), nest_limit - 1) + if self.pack_extended_type(obj): + # it means that obj was succesfully handled by + # handle_extended_type, so we are done + return if self._default is not None: return self._pack(self._default(obj), nest_limit - 1) raise TypeError("Cannot serialize %r" % obj) + def pack_extended_type(self, obj): + res = self.handle_extended_type(obj) + if res is None: + return False + fmt, typecode, data = res + # for now we support only this. We should add support for the other + # fixext/ext formats + assert fmt == "ext 32" + assert 0 <= typecode <= 127 + N = len(data) + self._buffer.write(struct.pack('>BIB', 0xc9, N, typecode)) + self._buffer.write(data) + return True + + def handle_extended_type(self, obj): + # by default we don't support any extended type. This can be + # overridden by subclasses + return None + def pack(self, obj): self._pack(obj) ret = self._buffer.getvalue() @@ -590,3 +578,52 @@ class Packer(object): def reset(self): self._buffer = StringIO() + + +def pack(o, stream, Packer=Packer, **kwargs): + """ + Pack object `o` and write it to `stream` + + See :class:`Packer` for options. + """ + packer = Packer(**kwargs) + stream.write(packer.pack(o)) + +def packb(o, Packer=Packer, **kwargs): + """ + Pack object `o` and return packed bytes + + See :class:`Packer` for options. + """ + return Packer(**kwargs).pack(o) + +def unpack(stream, Unpacker=Unpacker, **kwargs): + """ + Unpack an object from `stream`. + + Raises `ExtraData` when `packed` contains extra bytes. + See :class:`Unpacker` for options. + """ + unpacker = Unpacker(stream, **kwargs) + ret = unpacker._fb_unpack() + if unpacker._fb_got_extradata(): + raise ExtraData(ret, unpacker._fb_get_extradata()) + return ret + +def unpackb(packed, Unpacker=Unpacker, **kwargs): + """ + Unpack an object from `packed`. + + Raises `ExtraData` when `packed` contains extra bytes. + See :class:`Unpacker` for options. + """ + unpacker = Unpacker(None, **kwargs) + unpacker.feed(packed) + try: + ret = unpacker._fb_unpack() + except OutOfData: + raise UnpackValueError("Data is not enough.") + if unpacker._fb_got_extradata(): + raise ExtraData(ret, unpacker._fb_get_extradata()) + return ret + diff --git a/test/test_extension.py b/test/test_extension.py new file mode 100644 index 0000000..45e6027 --- /dev/null +++ b/test/test_extension.py @@ -0,0 +1,24 @@ +import array +import msgpack + +def test_extension_type(): + class MyPacker(msgpack.Packer): + def handle_extended_type(self, obj): + if isinstance(obj, array.array): + fmt = "ext 32" + typecode = 123 # application specific typecode + data = obj.tostring() + return fmt, typecode, data + + class MyUnpacker(msgpack.Unpacker): + def handle_extended_type(self, typecode, data): + assert typecode == 123 + obj = array.array('d') + obj.fromstring(data) + return obj + + obj = [42, 'hello', array.array('d', [1.1, 2.2, 3.3])] + s = msgpack.packb(obj, MyPacker) + obj2 = msgpack.unpackb(s, MyUnpacker) + assert obj == obj2 + From da12e177a31445492795f4003bbe0328645325b2 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 17 Oct 2013 08:35:08 +0900 Subject: [PATCH 0713/1172] Add bin type support. --- msgpack/_packer.pyx | 28 +++++++++++++++++++++------- msgpack/pack.h | 3 ++- msgpack/pack_template.h | 27 ++++++++++++++++++++++++++- msgpack/unpack.h | 9 +++++++++ msgpack/unpack_define.h | 17 ++++++----------- msgpack/unpack_template.h | 23 +++++++++++++++++------ 6 files changed, 81 insertions(+), 26 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 6289192..b9ae505 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -26,6 +26,7 @@ cdef extern from "pack.h": int msgpack_pack_array(msgpack_packer* pk, size_t l) int msgpack_pack_map(msgpack_packer* pk, size_t l) int msgpack_pack_raw(msgpack_packer* pk, size_t l) + int msgpack_pack_bin(msgpack_packer* pk, size_t l) int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) cdef int DEFAULT_RECURSE_LIMIT=511 @@ -56,6 +57,9 @@ cdef class Packer(object): :param bool autoreset: Reset buffer after each pack and return it's content as `bytes`. (default: True). If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. + :param bool use_bin_type: + Use bin type introduced in msgpack spec 2.0 for bytes. + It also enable str8 type for unicode. """ cdef msgpack_packer pk cdef object _default @@ -64,6 +68,7 @@ cdef class Packer(object): cdef char *encoding cdef char *unicode_errors cdef bool use_float + cdef bool use_bin_type cdef bint autoreset def __cinit__(self): @@ -74,11 +79,13 @@ cdef class Packer(object): self.pk.buf_size = buf_size self.pk.length = 0 - def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, bint autoreset=1): + def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', + use_single_float=False, bint autoreset=1, bint use_bin_type=0): """ """ self.use_float = use_single_float self.autoreset = autoreset + self.pk.use_bin_type = use_bin_type if default is not None: if not PyCallable_Check(default): raise TypeError("default must be a callable.") @@ -110,6 +117,7 @@ cdef class Packer(object): cdef char* rawval cdef int ret cdef dict d + cdef size_t L if nest_limit < 0: raise PackValueError("recursion limit exceeded.") @@ -140,9 +148,10 @@ cdef class Packer(object): ret = msgpack_pack_double(&self.pk, dval) elif PyBytes_Check(o): rawval = o - ret = msgpack_pack_raw(&self.pk, len(o)) + L = len(o) + ret = msgpack_pack_bin(&self.pk, L) if ret == 0: - ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) + ret = msgpack_pack_raw_body(&self.pk, rawval, L) elif PyUnicode_Check(o): if not self.encoding: raise TypeError("Can't encode unicode string: no encoding is specified") @@ -247,21 +256,26 @@ cdef class Packer(object): return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) -def pack(object o, object stream, default=None, str encoding='utf-8', str unicode_errors='strict'): +def pack(object o, object stream, + default=None, str encoding='utf-8', str unicode_errors='strict', + bint use_single_float=False, bint use_bin_type=False): """ pack an object `o` and write it to stream) See :class:`Packer` for options. """ - packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors) + packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors, + use_single_float=use_single_float, use_bin_type=use_bin_type) stream.write(packer.pack(o)) -def packb(object o, default=None, encoding='utf-8', str unicode_errors='strict', bint use_single_float=False): +def packb(object o, + default=None, str encoding='utf-8', str unicode_errors='strict', + bint use_single_float=False, bint use_bin_type=False): """ pack o and return packed bytes See :class:`Packer` for options. """ packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors, - use_single_float=use_single_float) + use_single_float=use_single_float, use_bin_type=use_bin_type) return packer.pack(o) diff --git a/msgpack/pack.h b/msgpack/pack.h index 1539991..001a0c1 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -34,11 +34,11 @@ typedef struct msgpack_packer { char *buf; size_t length; size_t buf_size; + bool use_bin_type; } msgpack_packer; typedef struct Packer Packer; -static inline int msgpack_pack_short(msgpack_packer* pk, short d); static inline int msgpack_pack_int(msgpack_packer* pk, int d); static inline int msgpack_pack_long(msgpack_packer* pk, long d); static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d); @@ -68,6 +68,7 @@ static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n); static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n); static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static inline int msgpack_pack_bin(msgpack_packer* pk, size_t l); static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 9e00d7e..ff1cbaa 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -667,7 +667,10 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l) if(l < 32) { unsigned char d = 0xa0 | (uint8_t)l; msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); - } else if(l < 65536) { + } else if (x->use_bin_type && l < 256) { // str8 is new format introduced with bin. + unsigned char buf[2] = {0xd9, (uint8_t)l}; + msgpack_pack_append_buffer(x, buf, 2); + } else if (l < 65536) { unsigned char buf[3]; buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l); msgpack_pack_append_buffer(x, buf, 3); @@ -678,6 +681,28 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l) } } +/* + * bin + */ +static inline int msgpack_pack_bin(msgpack_packer *x, size_t l) +{ + if (!x->use_bin_type) { + return msgpack_pack_raw(x, l) + } + if (l < 256) { + unsigned char buf[2] = {0xc4, (unsigned char)l}; + msgpack_pack_append_buffer(x, buf, 2); + } else if (l < 65536) { + unsigned char buf[3] = {0xc5}; + _msgpack_store16(&buf[1], (uint16_t)l); + msgpack_pack_append_buffer(x, buf, 3); + } else { + unsigned char buf[5] = {0xc6}; + _msgpack_store32(&buf[1], (uint32_t)l); + msgpack_pack_append_buffer(x, buf, 5); + } +} + static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t l) { msgpack_pack_append_buffer(x, (const unsigned char*)b, l); diff --git a/msgpack/unpack.h b/msgpack/unpack.h index baeed1f..03c735e 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -226,4 +226,13 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* return 0; } +static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) +{ + PyObject *py = PyBytes_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + return 0; +} + #include "unpack_template.h" diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index c81b990..0b14f52 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -34,6 +34,7 @@ extern "C" { #endif +// CS is first byte & 0x1f typedef enum { CS_HEADER = 0x00, // nil @@ -41,9 +42,9 @@ typedef enum { //CS_ = 0x02, // false //CS_ = 0x03, // true - //CS_ = 0x04, - //CS_ = 0x05, - //CS_ = 0x06, + CS_BIN_8 = 0x04, + CS_BIN_16 = 0x05, + CS_BIN_32 = 0x06, //CS_ = 0x07, //CS_ = 0x08, @@ -59,12 +60,7 @@ typedef enum { CS_INT_32 = 0x12, CS_INT_64 = 0x13, - //CS_ = 0x14, - //CS_ = 0x15, - //CS_BIG_INT_16 = 0x16, - //CS_BIG_INT_32 = 0x17, - //CS_BIG_FLOAT_16 = 0x18, - //CS_BIG_FLOAT_32 = 0x19, + CS_RAW_8 = 0x19, CS_RAW_16 = 0x1a, CS_RAW_32 = 0x1b, CS_ARRAY_16 = 0x1c, @@ -72,9 +68,8 @@ typedef enum { CS_MAP_16 = 0x1e, CS_MAP_32 = 0x1f, - //ACS_BIG_INT_VALUE, - //ACS_BIG_FLOAT_VALUE, ACS_RAW_VALUE, + ACS_BIN_VALUE, } msgpack_unpack_state; diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 29ac935..25229ac 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -151,8 +151,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l */ \ goto _header_again -#define NEXT_CS(p) \ - ((unsigned int)*p & 0x1f) +#define NEXT_CS(p) ((unsigned int)*p & 0x1f) #ifdef USE_CASE_RANGE #define SWITCH_RANGE_BEGIN switch(*p) { @@ -185,9 +184,6 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l push_simple_value(_false); case 0xc3: // true push_simple_value(_true); - //case 0xc4: - //case 0xc5: - //case 0xc6: //case 0xc7: //case 0xc8: //case 0xc9: @@ -202,12 +198,15 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l case 0xd2: // signed int 32 case 0xd3: // signed int 64 again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); + case 0xc4: // bin 8 + case 0xc5: // bin 16 + case 0xc6: // bin 32 //case 0xd4: //case 0xd5: //case 0xd6: // big integer 16 //case 0xd7: // big integer 32 //case 0xd8: // big float 16 - //case 0xd9: // big float 32 + case 0xd9: // raw 8 case 0xda: // raw 16 case 0xdb: // raw 32 case 0xdc: // array 16 @@ -290,6 +289,18 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l // // FIXME // push_variable_value(_big_float, data, n, trail); + case CS_BIN_8: + again_fixed_trail_if_zero(ACS_BIN_VALUE, *(uint8_t*)n, _bin_zero); + case CS_BIN_16: + again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load16(uint16_t,n), _bin_zero); + case CS_BIN_32: + again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load32(uint32_t,n), _bin_zero); + case ACS_BIN_VALUE: + _bin_zero: + push_variable_value(_bin, data, n, trail); + + case CS_RAW_8: + again_fixed_trail_if_zero(ACS_RAW_VALUE, *(uint8_t*)n, _raw_zero); case CS_RAW_16: again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero); case CS_RAW_32: From 171c5381135bdfc6db154100ad5a9913955408a7 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 17 Oct 2013 08:44:25 +0900 Subject: [PATCH 0714/1172] refactoring. --- msgpack/__init__.py | 22 ++++++++++++++++++++-- msgpack/_packer.pyx | 27 +-------------------------- msgpack/fallback.py | 17 ----------------- 3 files changed, 21 insertions(+), 45 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 77f6b81..5ce531e 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,14 +4,32 @@ from msgpack.exceptions import * import os if os.environ.get('MSGPACK_PUREPYTHON'): - from msgpack.fallback import pack, packb, Packer, unpack, unpackb, Unpacker + from msgpack.fallback import Packer, unpack, unpackb, Unpacker else: try: - from msgpack._packer import pack, packb, Packer + from msgpack._packer import Packer from msgpack._unpacker import unpack, unpackb, Unpacker except ImportError: from msgpack.fallback import pack, packb, Packer, unpack, unpackb, Unpacker + +def pack(o, stream, **kwargs): + """ + Pack object `o` and write it to `stream` + + See :class:`Packer` for options. + """ + packer = Packer(**kwargs) + stream.write(packer.pack(o)) + +def packb(o, **kwargs): + """ + Pack object `o` and return packed bytes + + See :class:`Packer` for options. + """ + return Packer(**kwargs).pack(o) + # alias for compatibility to simplejson/marshal/pickle. load = unpack loads = unpackb diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index b9ae505..7082445 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -13,6 +13,7 @@ cdef extern from "pack.h": char* buf size_t length size_t buf_size + bint use_bin_type int msgpack_pack_int(msgpack_packer* pk, int d) int msgpack_pack_nil(msgpack_packer* pk) @@ -68,7 +69,6 @@ cdef class Packer(object): cdef char *encoding cdef char *unicode_errors cdef bool use_float - cdef bool use_bin_type cdef bint autoreset def __cinit__(self): @@ -254,28 +254,3 @@ cdef class Packer(object): def bytes(self): """Return buffer content.""" return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) - - -def pack(object o, object stream, - default=None, str encoding='utf-8', str unicode_errors='strict', - bint use_single_float=False, bint use_bin_type=False): - """ - pack an object `o` and write it to stream) - - See :class:`Packer` for options. - """ - packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors, - use_single_float=use_single_float, use_bin_type=use_bin_type) - stream.write(packer.pack(o)) - -def packb(object o, - default=None, str encoding='utf-8', str unicode_errors='strict', - bint use_single_float=False, bint use_bin_type=False): - """ - pack o and return packed bytes - - See :class:`Packer` for options. - """ - packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors, - use_single_float=use_single_float, use_bin_type=use_bin_type) - return packer.pack(o) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 8f9d646..3bf0489 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -60,23 +60,6 @@ TYPE_RAW = 3 DEFAULT_RECURSE_LIMIT=511 -def pack(o, stream, **kwargs): - """ - Pack object `o` and write it to `stream` - - See :class:`Packer` for options. - """ - packer = Packer(**kwargs) - stream.write(packer.pack(o)) - -def packb(o, **kwargs): - """ - Pack object `o` and return packed bytes - - See :class:`Packer` for options. - """ - return Packer(**kwargs).pack(o) - def unpack(stream, **kwargs): """ Unpack an object from `stream`. From 84f6b100190049a5d1bceb916208eeae2a3d2591 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 17 Oct 2013 08:52:59 +0900 Subject: [PATCH 0715/1172] Add bin type support to pure Python packer. --- msgpack/fallback.py | 32 ++++++++++++++++++++++++-------- msgpack/pack_template.h | 2 +- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 3bf0489..49a2bc5 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -429,11 +429,15 @@ class Packer(object): :param bool autoreset: Reset buffer after each pack and return it's content as `bytes`. (default: True). If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. + :param bool use_bin_type: + Use bin type introduced in msgpack spec 2.0 for bytes. + It also enable str8 type for unicode. """ def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', - use_single_float=False, autoreset=True): + use_single_float=False, autoreset=True, use_bin_type=False): self._use_float = use_single_float self._autoreset = autoreset + self._use_bin_type = use_bin_type self._encoding = encoding self._unicode_errors = unicode_errors self._buffer = StringIO() @@ -473,6 +477,17 @@ class Packer(object): if -0x8000000000000000 <= obj < -0x80000000: return self._buffer.write(struct.pack(">Bq", 0xd3, obj)) raise PackValueError("Integer value out of range") + if self._use_bin_type and isinstance(obj, bytes): + n = len(obj) + if n <= 0xff: + self._buffer.write(struct.pack('>BB', 0xc4, n)) + elif n <= 0xffff: + self._buffer.write(struct.pack(">BH", 0xc5, n)) + elif n <= 0xffffffff: + self._buffer.write(struct.pack(">BI", 0xc6, n)) + else: + raise PackValueError("Bytes is too large") + return self._buffer.write(obj) if isinstance(obj, (Unicode, bytes)): if isinstance(obj, Unicode): if self._encoding is None: @@ -483,19 +498,20 @@ class Packer(object): n = len(obj) if n <= 0x1f: self._buffer.write(struct.pack('B', 0xa0 + n)) - return self._buffer.write(obj) - if n <= 0xffff: + elif self._use_bin_type and n <= 0xff: + self._buffer.write(struct.pack('>BB', 0xd9, n)) + elif n <= 0xffff: self._buffer.write(struct.pack(">BH", 0xda, n)) - return self._buffer.write(obj) - if n <= 0xffffffff: + elif n <= 0xffffffff: self._buffer.write(struct.pack(">BI", 0xdb, n)) - return self._buffer.write(obj) - raise PackValueError("String is too large") + else: + raise PackValueError("String is too large") + return self._buffer.write(obj) if isinstance(obj, float): if self._use_float: return self._buffer.write(struct.pack(">Bf", 0xca, obj)) return self._buffer.write(struct.pack(">Bd", 0xcb, obj)) - if isinstance(obj, list) or isinstance(obj, tuple): + if isinstance(obj, (list, tuple)): n = len(obj) self._fb_pack_array_header(n) for i in xrange(n): diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index ff1cbaa..d228d7a 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -664,7 +664,7 @@ static inline int msgpack_pack_map(msgpack_packer* x, unsigned int n) static inline int msgpack_pack_raw(msgpack_packer* x, size_t l) { - if(l < 32) { + if (l < 32) { unsigned char d = 0xa0 | (uint8_t)l; msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if (x->use_bin_type && l < 256) { // str8 is new format introduced with bin. From 85eaff344b74092ccf076ba2465e9bdb1537409b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 17 Oct 2013 09:15:19 +0900 Subject: [PATCH 0716/1172] Add bin type support for fallback Unpacker. --- msgpack/__init__.py | 2 +- msgpack/_unpacker.pyx | 1 - msgpack/fallback.py | 19 +++++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 5ce531e..7f0519e 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -10,7 +10,7 @@ else: from msgpack._packer import Packer from msgpack._unpacker import unpack, unpackb, Unpacker except ImportError: - from msgpack.fallback import pack, packb, Packer, unpack, unpackb, Unpacker + from msgpack.fallback import Packer, unpack, unpackb, Unpacker def pack(o, stream, **kwargs): diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 1f4dd85..7b0c8a6 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -18,7 +18,6 @@ from msgpack.exceptions import ( ) - cdef extern from "unpack.h": ctypedef struct msgpack_user: bint use_list diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 49a2bc5..901c6c8 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -57,6 +57,7 @@ TYPE_IMMEDIATE = 0 TYPE_ARRAY = 1 TYPE_MAP = 2 TYPE_RAW = 3 +TYPE_BIN = 4 DEFAULT_RECURSE_LIMIT=511 @@ -297,6 +298,10 @@ class Unpacker(object): obj = struct.unpack(">i", self._fb_read(4, write_bytes))[0] elif b == 0xd3: obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0] + elif b == 0xd9: + n = struct.unpack("B", self._fb_read(1, write_bytes))[0] + obj = self._fb_read(n, write_bytes) + typ = TYPE_RAW elif b == 0xda: n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] obj = self._fb_read(n, write_bytes) @@ -305,6 +310,18 @@ class Unpacker(object): n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] obj = self._fb_read(n, write_bytes) typ = TYPE_RAW + elif b == 0xc4: + n = struct.unpack("B", self._fb_read(1, write_bytes))[0] + obj = self._fb_read(n, write_bytes) + typ = TYPE_BIN + elif b == 0xc5: + n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + obj = self._fb_read(n, write_bytes) + typ = TYPE_BIN + elif b == 0xc6: + n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + obj = self._fb_read(n, write_bytes) + typ = TYPE_BIN elif b == 0xdc: n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] typ = TYPE_ARRAY @@ -373,6 +390,8 @@ class Unpacker(object): if self._encoding is not None: obj = obj.decode(self._encoding, self._unicode_errors) return obj + if typ == TYPE_BIN: + return obj assert typ == TYPE_IMMEDIATE return obj From f162bf6f79d7a50e10530e42630b063af3bb8eb1 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 17 Oct 2013 09:37:20 +0900 Subject: [PATCH 0717/1172] Add tests for str8 and bin types. --- test/test_newspec.py | 69 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 test/test_newspec.py diff --git a/test/test_newspec.py b/test/test_newspec.py new file mode 100644 index 0000000..8bc2cfe --- /dev/null +++ b/test/test_newspec.py @@ -0,0 +1,69 @@ +# coding: utf-8 + +from msgpack import packb, unpackb + + +def test_str8(): + header = b'\xd9' + data = b'x' * 32 + b = packb(data.decode(), use_bin_type=True) + assert len(b) == len(data) + 2 + assert b[0:2] == header + b'\x20' + assert b[2:] == data + assert unpackb(b) == data + + data = b'x' * 255 + b = packb(data.decode(), use_bin_type=True) + assert len(b) == len(data) + 2 + assert b[0:2] == header + b'\xff' + assert b[2:] == data + assert unpackb(b) == data + + +def test_bin8(): + header = b'\xc4' + data = b'' + b = packb(data, use_bin_type=True) + assert len(b) == len(data) + 2 + assert b[0:2] == header + b'\x00' + assert b[2:] == data + assert unpackb(b) == data + + data = b'x' * 255 + b = packb(data, use_bin_type=True) + assert len(b) == len(data) + 2 + assert b[0:2] == header + b'\xff' + assert b[2:] == data + assert unpackb(b) == data + + +def test_bin16(): + header = b'\xc5' + data = b'x' * 256 + b = packb(data, use_bin_type=True) + assert len(b) == len(data) + 3 + assert b[0:1] == header + assert b[1:3] == b'\x01\x00' + assert b[3:] == data + assert unpackb(b) == data + + data = b'x' * 65535 + b = packb(data, use_bin_type=True) + assert len(b) == len(data) + 3 + assert b[0:1] == header + assert b[1:3] == b'\xff\xff' + assert b[3:] == data + assert unpackb(b) == data + + +def test_bin32(): + header = b'\xc6' + data = b'x' * 65536 + b = packb(data, use_bin_type=True) + assert len(b) == len(data) + 5 + assert b[0:1] == header + assert b[1:5] == b'\x00\x01\x00\x00' + assert b[5:] == data + assert unpackb(b) == data + + From d9439204c77a0ab58248c0b78237f16c4496647c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 17 Oct 2013 11:29:36 +0900 Subject: [PATCH 0718/1172] Add ext type support to fallback.Unpacker. --- msgpack/__init__.py | 4 +++ msgpack/fallback.py | 72 +++++++++++++++++++++++++++++++++++---------- 2 files changed, 60 insertions(+), 16 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 7f0519e..79107b6 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -2,6 +2,10 @@ from msgpack._version import version from msgpack.exceptions import * +from collections import namedtuple + +ExtType = namedtuple('ExtType', 'code data') + import os if os.environ.get('MSGPACK_PUREPYTHON'): from msgpack.fallback import Packer, unpack, unpackb, Unpacker diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 901c6c8..074fcba 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -48,6 +48,9 @@ from msgpack.exceptions import ( PackValueError, ExtraData) +from msgpack import ExtType + + EX_SKIP = 0 EX_CONSTRUCT = 1 EX_READ_ARRAY_HEADER = 2 @@ -58,8 +61,9 @@ TYPE_ARRAY = 1 TYPE_MAP = 2 TYPE_RAW = 3 TYPE_BIN = 4 +TYPE_EXT = 5 -DEFAULT_RECURSE_LIMIT=511 +DEFAULT_RECURSE_LIMIT = 511 def unpack(stream, **kwargs): """ @@ -112,6 +116,9 @@ class Unpacker(object): should be callable and Unpacker calls it with a list of key-value pairs after deserializing a map. + `ext_hook` is callback for ext (User defined) type. It called with two + arguments: (code, bytes). default: `msgpack.ExtType` + `encoding` is the encoding used for decoding msgpack bytes. If it is None (default), msgpack bytes are deserialized to Python bytes. @@ -143,7 +150,8 @@ class Unpacker(object): def __init__(self, file_like=None, read_size=0, use_list=True, object_hook=None, object_pairs_hook=None, list_hook=None, - encoding=None, unicode_errors='strict', max_buffer_size=0): + encoding=None, unicode_errors='strict', max_buffer_size=0, + ext_hook=ExtType): if file_like is None: self._fb_feeding = True else: @@ -167,6 +175,7 @@ class Unpacker(object): self._list_hook = list_hook self._object_hook = object_hook self._object_pairs_hook = object_pairs_hook + self._ext_hook = ext_hook if list_hook is not None and not callable(list_hook): raise ValueError('`list_hook` is not callable') @@ -177,6 +186,8 @@ class Unpacker(object): if object_hook is not None and object_pairs_hook is not None: raise ValueError("object_pairs_hook and object_hook are mutually " "exclusive") + if not callable(ext_hook): + raise ValueError("`ext_hook` is not callable") def feed(self, next_bytes): if isinstance(next_bytes, array.array): @@ -278,6 +289,30 @@ class Unpacker(object): obj = False elif b == 0xc3: obj = True + elif b == 0xc4: + typ = TYPE_BIN + n = struct.unpack("B", self._fb_read(1, write_bytes))[0] + obj = self._fb_read(n, write_bytes) + elif b == 0xc5: + typ = TYPE_BIN + n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + obj = self._fb_read(n, write_bytes) + elif b == 0xc6: + typ = TYPE_BIN + n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + obj = self._fb_read(n, write_bytes) + elif b == 0xc7: # ext 8 + typ = TYPE_EXT + L, n = struct.unpack('Bb', self._fb_read(2, write_bytes)) + obj = self._fb_read(L, write_bytes) + elif b == 0xc8: # ext 16 + typ = TYPE_EXT + L, n = struct.unpack('>Hb', self._fb_read(3, write_bytes)) + obj = self._fb_read(L, write_bytes) + elif b == 0xc9: # ext 32 + typ = TYPE_EXT + L, n = struct.unpack('>Ib', self._fb_read(5, write_bytes)) + obj = self._fb_read(L, write_bytes) elif b == 0xca: obj = struct.unpack(">f", self._fb_read(4, write_bytes))[0] elif b == 0xcb: @@ -298,30 +333,33 @@ class Unpacker(object): obj = struct.unpack(">i", self._fb_read(4, write_bytes))[0] elif b == 0xd3: obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0] + elif b == 0xd4: # fixext 1 + typ = TYPE_EXT + n, obj = struct.unpack('b1s', self._fb_read(2, write_bytes)) + elif b == 0xd5: # fixext 2 + typ = TYPE_EXT + n, obj = struct.unpack('b2s', self._fb_read(3, write_bytes)) + elif b == 0xd6: # fixext 4 + typ = TYPE_EXT + n, obj = struct.unpack('b4s', self._fb_read(5, write_bytes)) + elif b == 0xd7: # fixext 8 + typ = TYPE_EXT + n, obj = struct.unpack('b8s', self._fb_read(9, write_bytes)) + elif b == 0xd8: # fixext 16 + typ = TYPE_EXT + n, obj = struct.unpack('b16s', self._fb_read(17, write_bytes)) elif b == 0xd9: + typ = TYPE_RAW n = struct.unpack("B", self._fb_read(1, write_bytes))[0] obj = self._fb_read(n, write_bytes) - typ = TYPE_RAW elif b == 0xda: + typ = TYPE_RAW n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] obj = self._fb_read(n, write_bytes) - typ = TYPE_RAW elif b == 0xdb: - n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] - obj = self._fb_read(n, write_bytes) typ = TYPE_RAW - elif b == 0xc4: - n = struct.unpack("B", self._fb_read(1, write_bytes))[0] - obj = self._fb_read(n, write_bytes) - typ = TYPE_BIN - elif b == 0xc5: - n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] - obj = self._fb_read(n, write_bytes) - typ = TYPE_BIN - elif b == 0xc6: n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] obj = self._fb_read(n, write_bytes) - typ = TYPE_BIN elif b == 0xdc: n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] typ = TYPE_ARRAY @@ -390,6 +428,8 @@ class Unpacker(object): if self._encoding is not None: obj = obj.decode(self._encoding, self._unicode_errors) return obj + if typ == TYPE_EXT: + return self._ext_hook(n, obj) if typ == TYPE_BIN: return obj assert typ == TYPE_IMMEDIATE From 5529dfe59660f3c2fc5058e6fa42b24fe764a255 Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Fri, 18 Oct 2013 14:38:52 +0200 Subject: [PATCH 0719/1172] kill some duplicate code from unpack/unpackb and move the logic to Unpacker.unpack_one. By doing this we no longer need to make the module-level pack/unpack parametric on the class, because they contain no logic at all --- msgpack/fallback.py | 95 +++++++++++++++++++++--------------------- test/test_extension.py | 8 ++-- 2 files changed, 52 insertions(+), 51 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index b7f455b..2c79482 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -62,6 +62,44 @@ EXTENDED_TYPE = 1000 DEFAULT_RECURSE_LIMIT=511 +def pack(o, stream, **kwargs): + """ + Pack object `o` and write it to `stream` + + See :class:`Packer` for options. + """ + packer = Packer(**kwargs) + stream.write(packer.pack(o)) + +def packb(o, **kwargs): + """ + Pack object `o` and return packed bytes + + See :class:`Packer` for options. + """ + return Packer(**kwargs).pack(o) + +def unpack(stream, **kwargs): + """ + Unpack an object from `stream`. + + Raises `ExtraData` when `packed` contains extra bytes. + See :class:`Unpacker` for options. + """ + unpacker = Unpacker(stream, **kwargs) + return unpacker.unpack_one() + +def unpackb(packed, **kwargs): + """ + Unpack an object from `packed`. + + Raises `ExtraData` when `packed` contains extra bytes. + See :class:`Unpacker` for options. + """ + unpacker = Unpacker(None, **kwargs) + unpacker.feed(packed) + return unpacker.unpack_one() + class Unpacker(object): """ Streaming unpacker. @@ -149,6 +187,15 @@ class Unpacker(object): raise ValueError("object_pairs_hook and object_hook are mutually " "exclusive") + def unpack_one(self): + try: + ret = self._fb_unpack() + except OutOfData: + raise UnpackValueError("Data is not enough.") + if self._fb_got_extradata(): + raise ExtraData(ret, self._fb_get_extradata()) + return ret + def feed(self, next_bytes): if isinstance(next_bytes, array.array): next_bytes = next_bytes.tostring() @@ -579,51 +626,3 @@ class Packer(object): def reset(self): self._buffer = StringIO() - -def pack(o, stream, Packer=Packer, **kwargs): - """ - Pack object `o` and write it to `stream` - - See :class:`Packer` for options. - """ - packer = Packer(**kwargs) - stream.write(packer.pack(o)) - -def packb(o, Packer=Packer, **kwargs): - """ - Pack object `o` and return packed bytes - - See :class:`Packer` for options. - """ - return Packer(**kwargs).pack(o) - -def unpack(stream, Unpacker=Unpacker, **kwargs): - """ - Unpack an object from `stream`. - - Raises `ExtraData` when `packed` contains extra bytes. - See :class:`Unpacker` for options. - """ - unpacker = Unpacker(stream, **kwargs) - ret = unpacker._fb_unpack() - if unpacker._fb_got_extradata(): - raise ExtraData(ret, unpacker._fb_get_extradata()) - return ret - -def unpackb(packed, Unpacker=Unpacker, **kwargs): - """ - Unpack an object from `packed`. - - Raises `ExtraData` when `packed` contains extra bytes. - See :class:`Unpacker` for options. - """ - unpacker = Unpacker(None, **kwargs) - unpacker.feed(packed) - try: - ret = unpacker._fb_unpack() - except OutOfData: - raise UnpackValueError("Data is not enough.") - if unpacker._fb_got_extradata(): - raise ExtraData(ret, unpacker._fb_get_extradata()) - return ret - diff --git a/test/test_extension.py b/test/test_extension.py index 45e6027..0a9c14f 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -18,7 +18,9 @@ def test_extension_type(): return obj obj = [42, 'hello', array.array('d', [1.1, 2.2, 3.3])] - s = msgpack.packb(obj, MyPacker) - obj2 = msgpack.unpackb(s, MyUnpacker) + packer = MyPacker() + unpacker = MyUnpacker(None) + s = packer.pack(obj) + unpacker.feed(s) + obj2 = unpacker.unpack_one() assert obj == obj2 - From 522c4bfc7993c296b78df9c9c91aac5fd40ae8e0 Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Fri, 18 Oct 2013 15:03:58 +0200 Subject: [PATCH 0720/1172] slightly change to API --- msgpack/fallback.py | 26 ++++++++++---------------- test/test_extension.py | 7 ++++--- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 2c79482..101bd0f 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -398,7 +398,7 @@ class Unpacker(object): if typ >= EXTENDED_TYPE: typ -= EXTENDED_TYPE data = self._fb_read(n, write_bytes) - return self.handle_extended_type(typ, data) + return self.read_extended_type(typ, data) assert typ == TYPE_IMMEDIATE return obj @@ -420,7 +420,7 @@ class Unpacker(object): self._fb_consume() return ret - def handle_extended_type(self, typecode, data): + def read_extended_type(self, typecode, data): raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode) def read_array_header(self, write_bytes=None): @@ -533,19 +533,19 @@ class Packer(object): if isinstance(obj, dict): return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj), nest_limit - 1) - if self.pack_extended_type(obj): - # it means that obj was succesfully handled by - # handle_extended_type, so we are done + if self.handle_unknown_type(obj): + # it means that obj was succesfully packed, so we are done return if self._default is not None: return self._pack(self._default(obj), nest_limit - 1) raise TypeError("Cannot serialize %r" % obj) - def pack_extended_type(self, obj): - res = self.handle_extended_type(obj) - if res is None: - return False - fmt, typecode, data = res + def handle_unknown_type(self, obj): + # by default we don't support any extended type. This can be + # overridden by subclasses + return None + + def pack_extended_type(self, fmt, typecode, data): # for now we support only this. We should add support for the other # fixext/ext formats assert fmt == "ext 32" @@ -553,12 +553,6 @@ class Packer(object): N = len(data) self._buffer.write(struct.pack('>BIB', 0xc9, N, typecode)) self._buffer.write(data) - return True - - def handle_extended_type(self, obj): - # by default we don't support any extended type. This can be - # overridden by subclasses - return None def pack(self, obj): self._pack(obj) diff --git a/test/test_extension.py b/test/test_extension.py index 0a9c14f..0b26f8e 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -3,15 +3,16 @@ import msgpack def test_extension_type(): class MyPacker(msgpack.Packer): - def handle_extended_type(self, obj): + def handle_unknown_type(self, obj): if isinstance(obj, array.array): fmt = "ext 32" typecode = 123 # application specific typecode data = obj.tostring() - return fmt, typecode, data + self.pack_extended_type(fmt, typecode, data) + return True class MyUnpacker(msgpack.Unpacker): - def handle_extended_type(self, typecode, data): + def read_extended_type(self, typecode, data): assert typecode == 123 obj = array.array('d') obj.fromstring(data) From c727440ba5fe2f77d6cc03171ad7c193a3f481ee Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Fri, 18 Oct 2013 15:45:50 +0200 Subject: [PATCH 0721/1172] automatically find the best format to encode extended types --- msgpack/fallback.py | 28 ++++++++++++++++++++++------ test/test_extension.py | 18 ++++++++++++++++-- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 101bd0f..f984dcd 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -545,13 +545,29 @@ class Packer(object): # overridden by subclasses return None - def pack_extended_type(self, fmt, typecode, data): - # for now we support only this. We should add support for the other - # fixext/ext formats - assert fmt == "ext 32" + def pack_extended_type(self, typecode, data): assert 0 <= typecode <= 127 - N = len(data) - self._buffer.write(struct.pack('>BIB', 0xc9, N, typecode)) + n = len(data) + if n == 1: + header = struct.pack(">BB", 0xd4, typecode) # fixext 1 + elif n == 2: + header = struct.pack(">BB", 0xd5, typecode) # fixext 2 + elif n == 4: + header = struct.pack(">BB", 0xd6, typecode) # fixext 4 + elif n == 8: + header = struct.pack(">BB", 0xd7, typecode) # fixext 8 + elif n == 16: + header = struct.pack(">BB", 0xd8, typecode) # fixext 16 + elif n <= 2**8-1: + header = struct.pack(">BBB", 0xc7, n, typecode) # ext 8 + elif n <= 2**16-1: + header = struct.pack(">BHB", 0xc8, n, typecode) # ext 16 + elif n <= 2**32-1: + header = struct.pack(">BIB", 0xc9, n, typecode) # ext 32 + else: + raise PackValueError("ext data too large") + # + self._buffer.write(header) self._buffer.write(data) def pack(self, obj): diff --git a/test/test_extension.py b/test/test_extension.py index 0b26f8e..1908fa2 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,14 +1,28 @@ import array +import struct import msgpack +def test_pack_extended_type(): + def p(s): + packer = msgpack.Packer() + packer.pack_extended_type(0x42, s) + return packer._buffer.getvalue() + assert p('A') == '\xd4\x42A' # fixext 1 + assert p('AB') == '\xd5\x42AB' # fixext 2 + assert p('ABCD') == '\xd6\x42ABCD' # fixext 4 + assert p('ABCDEFGH') == '\xd7\x42ABCDEFGH' # fixext 8 + assert p('A'*16) == '\xd8\x42' + 'A'*16 # fixext 16 + assert p('ABC') == '\xc7\x03\x42ABC' # ext 8 + assert p('A'*0x0123) == '\xc8\x01\x23\x42' + 'A'*0x0123 # ext 16 + assert p('A'*0x00012345) == '\xc9\x00\x01\x23\x45\x42' + 'A'*0x00012345 # ext 32 + def test_extension_type(): class MyPacker(msgpack.Packer): def handle_unknown_type(self, obj): if isinstance(obj, array.array): - fmt = "ext 32" typecode = 123 # application specific typecode data = obj.tostring() - self.pack_extended_type(fmt, typecode, data) + self.pack_extended_type(typecode, data) return True class MyUnpacker(msgpack.Unpacker): From afa28fb2051cb00f03c83e020745e1eb238ff4ac Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Fri, 18 Oct 2013 15:54:12 +0200 Subject: [PATCH 0722/1172] add support to unpack all ext formats --- msgpack/fallback.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index f984dcd..c272420 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -336,7 +336,33 @@ class Unpacker(object): elif b == 0xdf: n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] typ = TYPE_MAP - elif b == 0xc9: + elif b == 0xd4: # fixext 1 + typ = struct.unpack(">B", self._fb_read(1, write_bytes))[0] + n = 1 + typ += EXTENDED_TYPE + elif b == 0xd5: # fixext 2 + typ = struct.unpack(">B", self._fb_read(1, write_bytes))[0] + n = 2 + typ += EXTENDED_TYPE + elif b == 0xd6: # fixext 4 + typ = struct.unpack(">B", self._fb_read(1, write_bytes))[0] + n = 4 + typ += EXTENDED_TYPE + elif b == 0xd7: # fixext 8 + typ = struct.unpack(">B", self._fb_read(1, write_bytes))[0] + n = 8 + typ += EXTENDED_TYPE + elif b == 0xd8: # fixext 16 + typ = struct.unpack(">B", self._fb_read(1, write_bytes))[0] + n = 16 + typ += EXTENDED_TYPE + elif b == 0xc7: # ext 8 + n, typ = struct.unpack(">Bb", self._fb_read(2, write_bytes)) + typ += EXTENDED_TYPE + elif b == 0xc8: # ext 16 + n, typ = struct.unpack(">Hb", self._fb_read(3, write_bytes)) + typ += EXTENDED_TYPE + elif b == 0xc9: # ext 32 n, typ = struct.unpack(">Ib", self._fb_read(5, write_bytes)) typ += EXTENDED_TYPE else: From 5467515065b95496b9f5b9d842ffc73c9ccb806e Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Fri, 18 Oct 2013 17:33:54 +0200 Subject: [PATCH 0723/1172] implement Packer.pack_extended_type also in the cython version of the code --- msgpack/_packer.pyx | 6 +++++ msgpack/pack.h | 2 ++ msgpack/pack_template.h | 60 +++++++++++++++++++++++++++++++++++++++++ test/test_extension.py | 2 +- 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 6289192..985559c 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -5,6 +5,7 @@ from cpython cimport * from libc.stdlib cimport * from libc.string cimport * from libc.limits cimport * +from libc.stdint cimport int8_t from msgpack.exceptions import PackValueError @@ -27,6 +28,7 @@ cdef extern from "pack.h": int msgpack_pack_map(msgpack_packer* pk, size_t l) int msgpack_pack_raw(msgpack_packer* pk, size_t l) int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) + int msgpack_pack_ext(msgpack_packer* pk, int8_t typecode, size_t l) cdef int DEFAULT_RECURSE_LIMIT=511 @@ -193,6 +195,10 @@ cdef class Packer(object): self.pk.length = 0 return buf + def pack_extended_type(self, typecode, data): + msgpack_pack_ext(&self.pk, typecode, len(data)) + msgpack_pack_raw_body(&self.pk, data, len(data)) + def pack_array_header(self, size_t size): cdef int ret = msgpack_pack_array(&self.pk, size) if ret == -1: diff --git a/msgpack/pack.h b/msgpack/pack.h index 1539991..08fdd82 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -70,6 +70,8 @@ static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n); static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); +static inline int msgpack_pack_ext(msgpack_packer* pk, int8_t typecode, size_t l); + static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) { char* buf = pk->buf; diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 9e00d7e..ac9815f 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -683,6 +683,66 @@ static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t msgpack_pack_append_buffer(x, (const unsigned char*)b, l); } +/* + * Ext + */ + +static inline int msgpack_pack_ext(msgpack_packer* x, int8_t typecode, size_t l) +{ + if (l == 1) { + unsigned char buf[2]; + buf[0] = 0xd4; + buf[1] = (unsigned char)typecode; + msgpack_pack_append_buffer(x, buf, 2); + } + else if(l == 2) { + unsigned char buf[2]; + buf[0] = 0xd5; + buf[1] = (unsigned char)typecode; + msgpack_pack_append_buffer(x, buf, 2); + } + else if(l == 4) { + unsigned char buf[2]; + buf[0] = 0xd6; + buf[1] = (unsigned char)typecode; + msgpack_pack_append_buffer(x, buf, 2); + } + else if(l == 8) { + unsigned char buf[2]; + buf[0] = 0xd7; + buf[1] = (unsigned char)typecode; + msgpack_pack_append_buffer(x, buf, 2); + } + else if(l == 16) { + unsigned char buf[2]; + buf[0] = 0xd8; + buf[1] = (unsigned char)typecode; + msgpack_pack_append_buffer(x, buf, 2); + } + else if(l < 256) { + unsigned char buf[3]; + buf[0] = 0xc7; + buf[1] = l; + buf[2] = (unsigned char)typecode; + msgpack_pack_append_buffer(x, buf, 3); + } else if(l < 65536) { + unsigned char buf[4]; + buf[0] = 0xc8; + _msgpack_store16(&buf[1], (uint16_t)l); + buf[3] = (unsigned char)typecode; + msgpack_pack_append_buffer(x, buf, 4); + } else { + unsigned char buf[6]; + buf[0] = 0xc9; + _msgpack_store32(&buf[1], (uint32_t)l); + buf[5] = (unsigned char)typecode; + msgpack_pack_append_buffer(x, buf, 6); + } + +} + + + #undef msgpack_pack_append_buffer #undef TAKE8_8 diff --git a/test/test_extension.py b/test/test_extension.py index 1908fa2..9ec1153 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -6,7 +6,7 @@ def test_pack_extended_type(): def p(s): packer = msgpack.Packer() packer.pack_extended_type(0x42, s) - return packer._buffer.getvalue() + return packer.bytes() assert p('A') == '\xd4\x42A' # fixext 1 assert p('AB') == '\xd5\x42AB' # fixext 2 assert p('ABCD') == '\xd6\x42ABCD' # fixext 4 From a7485eccb2e5fcebbd76612a658f2e18bdebe745 Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Fri, 18 Oct 2013 17:46:42 +0200 Subject: [PATCH 0724/1172] add the hook for unknown types also to the cython Packer --- msgpack/_packer.pyx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 985559c..f2a0f76 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -176,6 +176,9 @@ cdef class Packer(object): for v in o: ret = self._pack(v, nest_limit-1) if ret != 0: break + elif self.handle_unknown_type(o): + # it means that obj was succesfully packed, so we are done + return 0 elif self._default: o = self._default(o) ret = self._pack(o, nest_limit-1) @@ -195,6 +198,9 @@ cdef class Packer(object): self.pk.length = 0 return buf + def handle_unknown_type(self, obj): + return None + def pack_extended_type(self, typecode, data): msgpack_pack_ext(&self.pk, typecode, len(data)) msgpack_pack_raw_body(&self.pk, data, len(data)) From ff858387d37d37ec4472f6b6ac7010d8f2b0744f Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Sat, 19 Oct 2013 01:49:03 +0200 Subject: [PATCH 0725/1172] implement unpack_one also for the cython version, and add a test for it --- msgpack/_unpacker.pyx | 18 ++++++++++++++++++ test/test_sequnpack.py | 15 ++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 1f4dd85..e05b9ed 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -359,6 +359,24 @@ cdef class Unpacker(object): """ return self._unpack(unpack_construct, write_bytes) + def unpack_one(self, object write_bytes=None): + """ + unpack one object + + If write_bytes is not None, it will be called with parts of the raw + message as it is unpacked. + + Raises `UnpackValueError` if there are no more bytes to unpack. + Raises ``ExtraData`` if there are still bytes left after the unpacking. + """ + try: + result = self.unpack() + except OutOfData: + raise UnpackValueError("Data is not enough") + if self.buf_head < self.buf_tail: + raise ExtraData(result, self.buf[self.buf_head:]) + return result + def skip(self, object write_bytes=None): """ read and ignore one object, returning None diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 9db14ca..abc447a 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,9 +1,10 @@ #!/usr/bin/env python # coding: utf-8 +import py import six from msgpack import Unpacker, BufferFull -from msgpack.exceptions import OutOfData +from msgpack.exceptions import OutOfData, ExtraData, UnpackValueError from pytest import raises @@ -85,3 +86,15 @@ def test_readbytes(): assert unpacker.unpack() == ord(b'a') assert unpacker.unpack() == ord(b'r') +def test_unpack_one(): + unpacker = Unpacker() + unpacker.feed('\xda\x00\x03abc') + assert unpacker.unpack_one() == 'abc' + # + unpacker = Unpacker() + unpacker.feed('\xda\x00\x03abcd') + py.test.raises(ExtraData, "unpacker.unpack_one()") + # + unpacker = Unpacker() + unpacker.feed('\xda\x00\x03ab') + py.test.raises(UnpackValueError, "unpacker.unpack_one()") From 985d4c1496d8c9186079ebc4e42aee319e67c385 Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Sat, 19 Oct 2013 11:34:28 +0200 Subject: [PATCH 0726/1172] add a test for unpacking extended types --- test/test_extension.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/test_extension.py b/test/test_extension.py index 9ec1153..96944a3 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -16,6 +16,28 @@ def test_pack_extended_type(): assert p('A'*0x0123) == '\xc8\x01\x23\x42' + 'A'*0x0123 # ext 16 assert p('A'*0x00012345) == '\xc9\x00\x01\x23\x45\x42' + 'A'*0x00012345 # ext 32 +def test_unpack_extended_type(): + class MyUnpacker(msgpack.Unpacker): + def read_extended_type(self, typecode, data): + return (typecode, data) + + def u(s): + unpacker = MyUnpacker() + unpacker.feed(s) + return unpacker.unpack_one() + + assert u('\xd4\x42A') == (0x42, 'A') # fixext 1 + assert u('\xd5\x42AB') == (0x42, 'AB') # fixext 2 + assert u('\xd6\x42ABCD') == (0x42, 'ABCD') # fixext 4 + assert u('\xd7\x42ABCDEFGH') == (0x42, 'ABCDEFGH') # fixext 8 + assert u('\xd8\x42' + 'A'*16) == (0x42, 'A'*16) # fixext 16 + assert u('\xc7\x03\x42ABC') == (0x42, 'ABC') # ext 8 + assert (u('\xc8\x01\x23\x42' + 'A'*0x0123) == + (0x42, 'A'*0x0123)) # ext 16 + assert (u('\xc9\x00\x01\x23\x45\x42' + 'A'*0x00012345) == + (0x42, 'A'*0x00012345)) # ext 32 + + def test_extension_type(): class MyPacker(msgpack.Packer): def handle_unknown_type(self, obj): From 56dd1650a42a454027ba335b494100a9f211758e Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Sat, 19 Oct 2013 17:27:16 +0200 Subject: [PATCH 0727/1172] implement unpacking for all the fixtext formats --- msgpack/_unpacker.pyx | 21 +++++++++++++++++++-- msgpack/unpack.h | 18 ++++++++++++++++++ msgpack/unpack_define.h | 14 ++++++++------ msgpack/unpack_template.h | 22 +++++++++++++++------- setup.py | 1 + test/test_extension.py | 1 + 6 files changed, 62 insertions(+), 15 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index e05b9ed..6500ef7 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -25,6 +25,7 @@ cdef extern from "unpack.h": PyObject* object_hook bint has_pairs_hook # call object_hook with k-v pairs PyObject* list_hook + PyObject* ext_type_hook char *encoding char *unicode_errors @@ -46,6 +47,7 @@ cdef extern from "unpack.h": cdef inline init_ctx(unpack_context *ctx, object object_hook, object object_pairs_hook, object list_hook, + object ext_type_hook, bint use_list, char* encoding, char* unicode_errors): unpack_init(ctx) ctx.user.use_list = use_list @@ -72,9 +74,17 @@ cdef inline init_ctx(unpack_context *ctx, raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook + if ext_type_hook is not None: + if not PyCallable_Check(ext_type_hook): + raise TypeError("ext_type_hook must be a callable.") + ctx.user.ext_type_hook = ext_type_hook + ctx.user.encoding = encoding ctx.user.unicode_errors = unicode_errors +def default_read_extended_type(typecode, data): + raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode) + def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=1, encoding=None, unicode_errors="strict", object_pairs_hook=None, @@ -107,7 +117,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, unicode_errors = unicode_errors.encode('ascii') cerr = PyBytes_AsString(unicode_errors) - init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) + init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, default_read_extended_type, + use_list, cenc, cerr) ret = unpack_construct(&ctx, buf, buf_len, &off) if ret == 1: obj = unpack_data(&ctx) @@ -249,7 +260,10 @@ cdef class Unpacker(object): self.unicode_errors = unicode_errors cerr = PyBytes_AsString(self.unicode_errors) - init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) + ext_type_hook = self.read_extended_type + Py_INCREF(ext_type_hook) + init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, + ext_type_hook, use_list, cenc, cerr) def feed(self, object next_bytes): """Append `next_bytes` to internal buffer.""" @@ -404,6 +418,9 @@ cdef class Unpacker(object): """ return self._unpack(read_map_header, write_bytes) + def read_extended_type(self, typecode, data): + return default_read_extended_type(typecode, data) + def __iter__(self): return self diff --git a/msgpack/unpack.h b/msgpack/unpack.h index baeed1f..97ebd3f 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -24,6 +24,7 @@ typedef struct unpack_user { PyObject *object_hook; bool has_pairs_hook; PyObject *list_hook; + PyObject *ext_type_hook; const char *encoding; const char *unicode_errors; } unpack_user; @@ -226,4 +227,21 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* return 0; } +static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos, + unsigned int lenght, msgpack_unpack_object* o) +{ + PyObject *py; + int8_t typecode = (int8_t)*pos++; + if (!u->ext_type_hook) { + PyErr_SetString(PyExc_AssertionError, "u->ext_type_hook cannot be NULL"); + return -1; + } + // lenght also includes the typecode, so the actual data is lenght-1 + py = PyEval_CallFunction(u->ext_type_hook, "(is#)", typecode, pos, lenght-1); + if (!py) + return -1; + *o = py; + return 0; +} + #include "unpack_template.h" diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index c81b990..986fa91 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -59,12 +59,12 @@ typedef enum { CS_INT_32 = 0x12, CS_INT_64 = 0x13, - //CS_ = 0x14, - //CS_ = 0x15, - //CS_BIG_INT_16 = 0x16, - //CS_BIG_INT_32 = 0x17, - //CS_BIG_FLOAT_16 = 0x18, - //CS_BIG_FLOAT_32 = 0x19, + CS_FIXEXT1 = 0x14, + CS_FIXEXT2 = 0x15, + CS_FIXEXT4 = 0x16, + CS_FIXEXT8 = 0x17, + CS_FIXEXT16 = 0x18, + CS_RAW_16 = 0x1a, CS_RAW_32 = 0x1b, CS_ARRAY_16 = 0x1c, @@ -75,6 +75,8 @@ typedef enum { //ACS_BIG_INT_VALUE, //ACS_BIG_FLOAT_VALUE, ACS_RAW_VALUE, + ACS_EXT_VALUE, + } msgpack_unpack_state; diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 29ac935..b051075 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -202,12 +202,16 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l case 0xd2: // signed int 32 case 0xd3: // signed int 64 again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); - //case 0xd4: - //case 0xd5: - //case 0xd6: // big integer 16 - //case 0xd7: // big integer 32 - //case 0xd8: // big float 16 - //case 0xd9: // big float 32 + case 0xd4: // fixext 1 + case 0xd5: // fixext 2 + case 0xd6: // fixext 4 + case 0xd7: // fixext 8 + again_fixed_trail_if_zero(ACS_EXT_VALUE, + (1 << (((unsigned int)*p) & 0x03))+1, + _ext_zero); + case 0xd8: // fixext 16 + again_fixed_trail_if_zero(ACS_EXT_VALUE, 16+1, _ext_zero); + //case 0xd9: case 0xda: // raw 16 case 0xdb: // raw 32 case 0xdc: // array 16 @@ -298,6 +302,10 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l _raw_zero: push_variable_value(_raw, data, n, trail); + case ACS_EXT_VALUE: + _ext_zero: + push_variable_value(_ext, data, n, trail); + case CS_ARRAY_16: start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM); case CS_ARRAY_32: @@ -309,7 +317,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l case CS_MAP_32: /* FIXME security guard */ start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); - + default: goto _failed; } diff --git a/setup.py b/setup.py index 1055a61..83ae79f 100644 --- a/setup.py +++ b/setup.py @@ -92,6 +92,7 @@ if not hasattr(sys, 'pypy_version_info'): libraries=libraries, include_dirs=['.'], define_macros=macros, + extra_compile_args=['-O0'], )) del libraries, macros diff --git a/test/test_extension.py b/test/test_extension.py index 96944a3..94117e1 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,3 +1,4 @@ +import py import array import struct import msgpack From c9b97f078854dda02dc9404ee9d88ca5e16fb493 Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Sat, 19 Oct 2013 18:04:30 +0200 Subject: [PATCH 0728/1172] implement unpacking of ext 8,16,32 --- msgpack/unpack_define.h | 17 ++++++++--------- msgpack/unpack_template.h | 21 ++++++++++++++++----- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 986fa91..17c287e 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -44,10 +44,9 @@ typedef enum { //CS_ = 0x04, //CS_ = 0x05, //CS_ = 0x06, - //CS_ = 0x07, - - //CS_ = 0x08, - //CS_ = 0x09, + CS_EXT_8 = 0x07, + CS_EXT_16 = 0x08, + CS_EXT_32 = 0x09, CS_FLOAT = 0x0a, CS_DOUBLE = 0x0b, CS_UINT_8 = 0x0c, @@ -59,11 +58,11 @@ typedef enum { CS_INT_32 = 0x12, CS_INT_64 = 0x13, - CS_FIXEXT1 = 0x14, - CS_FIXEXT2 = 0x15, - CS_FIXEXT4 = 0x16, - CS_FIXEXT8 = 0x17, - CS_FIXEXT16 = 0x18, + //CS_FIXEXT1 = 0x14, + //CS_FIXEXT2 = 0x15, + //CS_FIXEXT4 = 0x16, + //CS_FIXEXT8 = 0x17, + //CS_FIXEXT16 = 0x18, CS_RAW_16 = 0x1a, CS_RAW_32 = 0x1b, diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index b051075..0c6af0e 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -188,9 +188,12 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l //case 0xc4: //case 0xc5: //case 0xc6: - //case 0xc7: - //case 0xc8: - //case 0xc9: + case 0xc7: // ext 8 + again_fixed_trail(NEXT_CS(p), 1); + case 0xc8: // ext 16 + again_fixed_trail(NEXT_CS(p), 2); + case 0xc9: // ext 32 + again_fixed_trail(NEXT_CS(p), 4); case 0xca: // float case 0xcb: // double case 0xcc: // unsigned int 8 @@ -242,8 +245,16 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l if((size_t)(pe - p) < trail) { goto _out; } n = p; p += trail - 1; switch(cs) { - //case CS_ - //case CS_ + case CS_EXT_8: + again_fixed_trail_if_zero(ACS_EXT_VALUE, *(uint8_t*)n+1, _ext_zero); + case CS_EXT_16: + again_fixed_trail_if_zero(ACS_EXT_VALUE, + _msgpack_load16(uint16_t,n)+1, + _ext_zero); + case CS_EXT_32: + again_fixed_trail_if_zero(ACS_EXT_VALUE, + _msgpack_load32(uint32_t,n)+1, + _ext_zero); case CS_FLOAT: { union { uint32_t i; float f; } mem; mem.i = _msgpack_load32(uint32_t,n); From 6386481024ec045d9ef991a2c975902276812508 Mon Sep 17 00:00:00 2001 From: Antonio Cuni Date: Sat, 19 Oct 2013 18:43:16 +0200 Subject: [PATCH 0729/1172] add a note in the README --- README.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.rst b/README.rst index 294cd63..600d7f7 100644 --- a/README.rst +++ b/README.rst @@ -140,6 +140,14 @@ It is also possible to pack/unpack custom data types. Here is an example for ``object_pairs_hook`` callback may instead be used to receive a list of key-value pairs. +Extended types +^^^^^^^^^^^^^^^ + +It is also possible to pack/unpack custom data types using the msgpack feature +of "extended types". For example, msgpack-pypy uses it to provide very fast serialization of int/float lists on top of PyPy (experimental for now): + +https://bitbucket.org/antocuni/msgpack-pypy/src/default/msgpack_pypy.py + Advanced unpacking control ^^^^^^^^^^^^^^^^^^^^^^^^^^ From 7123341ca89a9a3afee8521cc16a1a419ea8871e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 20 Oct 2013 14:34:36 +0900 Subject: [PATCH 0730/1172] code refactoring. --- msgpack/fallback.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 074fcba..dfaaa54 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -163,12 +163,10 @@ class Unpacker(object): self._fb_buf_o = 0 self._fb_buf_i = 0 self._fb_buf_n = 0 - self._max_buffer_size = (2**31-1 if max_buffer_size == 0 - else max_buffer_size) - self._read_size = (read_size if read_size != 0 - else min(self._max_buffer_size, 2048)) + self._max_buffer_size = max_buffer_size or 2**31-1 if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") + self._read_size = read_size or min(self._max_buffer_size, 2048) self._encoding = encoding self._unicode_errors = unicode_errors self._use_list = use_list @@ -234,26 +232,26 @@ class Unpacker(object): return b''.join(bufs) def _fb_read(self, n, write_bytes=None): - if (write_bytes is None and self._fb_buf_i < len(self._fb_buffers) - and self._fb_buf_o + n < len(self._fb_buffers[self._fb_buf_i])): + buffs = self._fb_buffers + if (write_bytes is None and self._fb_buf_i < len(buffs) and + self._fb_buf_o + n < len(buffs[self._fb_buf_i])): self._fb_buf_o += n - return self._fb_buffers[self._fb_buf_i][ - self._fb_buf_o-n:self._fb_buf_o] + return buffs[self._fb_buf_i][self._fb_buf_o - n:self._fb_buf_o] + ret = b'' while len(ret) != n: - if self._fb_buf_i == len(self._fb_buffers): + if self._fb_buf_i == len(buffs): if self._fb_feeding: break tmp = self.file_like.read(self._read_size) if not tmp: break - self._fb_buffers.append(tmp) + buffs.append(tmp) continue sliced = n - len(ret) - ret += self._fb_buffers[self._fb_buf_i][ - self._fb_buf_o:self._fb_buf_o + sliced] + ret += buffs[self._fb_buf_i][self._fb_buf_o:self._fb_buf_o + sliced] self._fb_buf_o += sliced - if self._fb_buf_o >= len(self._fb_buffers[self._fb_buf_i]): + if self._fb_buf_o >= len(buffs[self._fb_buf_i]): self._fb_buf_o = 0 self._fb_buf_i += 1 if len(ret) != n: @@ -410,10 +408,9 @@ class Unpacker(object): return if self._object_pairs_hook is not None: ret = self._object_pairs_hook( - (self._fb_unpack(EX_CONSTRUCT, write_bytes), - self._fb_unpack(EX_CONSTRUCT, write_bytes)) - for _ in xrange(n) - ) + (self._fb_unpack(EX_CONSTRUCT, write_bytes), + self._fb_unpack(EX_CONSTRUCT, write_bytes)) + for _ in xrange(n)) else: ret = {} for _ in xrange(n): From aa68c9b8330b130d600b22ec47d5c3841499b536 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 20 Oct 2013 15:40:20 +0900 Subject: [PATCH 0731/1172] fallback: Support pack_ext_type. --- msgpack/_packer.pyx | 2 +- msgpack/_unpacker.pyx | 2 -- msgpack/fallback.py | 46 +++++++++++++++++++++++++++++++++++------- test/test_extension.py | 26 ++++++++++++------------ 4 files changed, 53 insertions(+), 23 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index df670ed..f033263 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -210,7 +210,7 @@ cdef class Packer(object): def handle_unknown_type(self, obj): return None - def pack_extended_type(self, typecode, data): + def pack_ext_type(self, typecode, data): msgpack_pack_ext(&self.pk, typecode, len(data)) msgpack_pack_raw_body(&self.pk, data, len(data)) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index cf30670..b0e66db 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -32,8 +32,6 @@ cdef extern from "unpack.h": msgpack_user user PyObject* obj size_t count - unsigned int ct - PyObject* key ctypedef int (*execute_fn)(unpack_context* ctx, const char* data, size_t len, size_t* off) except? -1 diff --git a/msgpack/fallback.py b/msgpack/fallback.py index dfaaa54..0b29700 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -42,11 +42,11 @@ else: newlist_hint = lambda size: [] from msgpack.exceptions import ( - BufferFull, - OutOfData, - UnpackValueError, - PackValueError, - ExtraData) + BufferFull, + OutOfData, + UnpackValueError, + PackValueError, + ExtraData) from msgpack import ExtType @@ -65,6 +65,7 @@ TYPE_EXT = 5 DEFAULT_RECURSE_LIMIT = 511 + def unpack(stream, **kwargs): """ Unpack an object from `stream`. @@ -78,6 +79,7 @@ def unpack(stream, **kwargs): raise ExtraData(ret, unpacker._fb_get_extradata()) return ret + def unpackb(packed, **kwargs): """ Unpack an object from `packed`. @@ -95,6 +97,7 @@ def unpackb(packed, **kwargs): raise ExtraData(ret, unpacker._fb_get_extradata()) return ret + class Unpacker(object): """ Streaming unpacker. @@ -548,8 +551,8 @@ class Packer(object): if isinstance(obj, Unicode): if self._encoding is None: raise TypeError( - "Can't encode unicode string: " - "no encoding is specified") + "Can't encode unicode string: " + "no encoding is specified") obj = obj.encode(self._encoding, self._unicode_errors) n = len(obj) if n <= 0x1f: @@ -616,6 +619,35 @@ class Packer(object): self._buffer = StringIO(ret) return ret + def pack_ext_type(self, typecode, data): + if not isinstance(typecode, int): + raise TypeError("typecode must have int type.") + if not 0 <= typecode <= 127: + raise ValueError("typecode should be 0-127") + if not isinstance(data, bytes): + raise TypeError("data must have bytes type") + L = len(data) + if L > 0xffffffff: + raise ValueError("Too large data") + if L == 1: + self._buffer.write(b'\xd4') + elif L == 2: + self._buffer.write(b'\xd5') + elif L == 4: + self._buffer.write(b'\xd6') + elif L == 8: + self._buffer.write(b'\xd7') + elif L == 16: + self._buffer.write(b'\xd8') + elif L <= 0xff: + self._buffer.write(b'\xc7' + struct.pack('B', L)) + elif L <= 0xffff: + self._buffer.write(b'\xc8' + struct.pack('>H', L)) + else: + self._buffer.write(b'\xc9' + struct.pack('>I', L)) + self._buffer.write(struct.pack('B', typecode)) + self._buffer.write(data) + def _fb_pack_array_header(self, n): if n <= 0x0f: return self._buffer.write(struct.pack('B', 0x90 + n)) diff --git a/test/test_extension.py b/test/test_extension.py index 94117e1..f2fa363 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,21 +1,21 @@ -import py import array -import struct import msgpack -def test_pack_extended_type(): + +def test_pack_ext_type(): def p(s): packer = msgpack.Packer() - packer.pack_extended_type(0x42, s) + packer.pack_ext_type(0x42, s) return packer.bytes() - assert p('A') == '\xd4\x42A' # fixext 1 - assert p('AB') == '\xd5\x42AB' # fixext 2 - assert p('ABCD') == '\xd6\x42ABCD' # fixext 4 - assert p('ABCDEFGH') == '\xd7\x42ABCDEFGH' # fixext 8 - assert p('A'*16) == '\xd8\x42' + 'A'*16 # fixext 16 - assert p('ABC') == '\xc7\x03\x42ABC' # ext 8 - assert p('A'*0x0123) == '\xc8\x01\x23\x42' + 'A'*0x0123 # ext 16 - assert p('A'*0x00012345) == '\xc9\x00\x01\x23\x45\x42' + 'A'*0x00012345 # ext 32 + assert p(b'A') == b'\xd4\x42A' # fixext 1 + assert p(b'AB') == b'\xd5\x42AB' # fixext 2 + assert p(b'ABCD') == b'\xd6\x42ABCD' # fixext 4 + assert p(b'ABCDEFGH') == b'\xd7\x42ABCDEFGH' # fixext 8 + assert p(b'A'*16) == b'\xd8\x42' + 'A'*16 # fixext 16 + assert p(b'ABC') == b'\xc7\x03\x42ABC' # ext 8 + assert p(b'A'*0x0123) == b'\xc8\x01\x23\x42' + b'A'*0x0123 # ext 16 + assert p(b'A'*0x00012345) == b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345 # ext 32 + def test_unpack_extended_type(): class MyUnpacker(msgpack.Unpacker): @@ -45,7 +45,7 @@ def test_extension_type(): if isinstance(obj, array.array): typecode = 123 # application specific typecode data = obj.tostring() - self.pack_extended_type(typecode, data) + self.pack_ext_type(typecode, data) return True class MyUnpacker(msgpack.Unpacker): From 96bcd76f49afd00f5b7def1ff7cfd002a7fa477d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 20 Oct 2013 20:28:32 +0900 Subject: [PATCH 0732/1172] Packing ExtType and some cleanup --- msgpack/__init__.py | 2 +- msgpack/_packer.pyx | 150 +++++++++++++++++++++------------------- msgpack/_unpacker.pyx | 49 ++++--------- msgpack/pack_template.h | 3 +- msgpack/unpack.h | 10 +-- test/test_extension.py | 71 +++++++++---------- test/test_sequnpack.py | 13 ---- 7 files changed, 130 insertions(+), 168 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 79107b6..a7b47b1 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -26,6 +26,7 @@ def pack(o, stream, **kwargs): packer = Packer(**kwargs) stream.write(packer.pack(o)) + def packb(o, **kwargs): """ Pack object `o` and return packed bytes @@ -40,4 +41,3 @@ loads = unpackb dump = pack dumps = packb - diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index f033263..f63667c 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -8,6 +8,8 @@ from libc.limits cimport * from libc.stdint cimport int8_t from msgpack.exceptions import PackValueError +from msgpack import ExtType + cdef extern from "pack.h": struct msgpack_packer: @@ -120,80 +122,87 @@ cdef class Packer(object): cdef int ret cdef dict d cdef size_t L + cdef int default_used = 0 if nest_limit < 0: raise PackValueError("recursion limit exceeded.") - if o is None: - ret = msgpack_pack_nil(&self.pk) - elif isinstance(o, bool): - if o: - ret = msgpack_pack_true(&self.pk) - else: - ret = msgpack_pack_false(&self.pk) - elif PyLong_Check(o): - if o > 0: - ullval = o - ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) - else: - llval = o - ret = msgpack_pack_long_long(&self.pk, llval) - elif PyInt_Check(o): - longval = o - ret = msgpack_pack_long(&self.pk, longval) - elif PyFloat_Check(o): - if self.use_float: - fval = o - ret = msgpack_pack_float(&self.pk, fval) - else: - dval = o - ret = msgpack_pack_double(&self.pk, dval) - elif PyBytes_Check(o): - rawval = o - L = len(o) - ret = msgpack_pack_bin(&self.pk, L) - if ret == 0: + while True: + if o is None: + ret = msgpack_pack_nil(&self.pk) + elif isinstance(o, bool): + if o: + ret = msgpack_pack_true(&self.pk) + else: + ret = msgpack_pack_false(&self.pk) + elif PyLong_Check(o): + if o > 0: + ullval = o + ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) + else: + llval = o + ret = msgpack_pack_long_long(&self.pk, llval) + elif PyInt_Check(o): + longval = o + ret = msgpack_pack_long(&self.pk, longval) + elif PyFloat_Check(o): + if self.use_float: + fval = o + ret = msgpack_pack_float(&self.pk, fval) + else: + dval = o + ret = msgpack_pack_double(&self.pk, dval) + elif PyBytes_Check(o): + rawval = o + L = len(o) + ret = msgpack_pack_bin(&self.pk, L) + if ret == 0: + ret = msgpack_pack_raw_body(&self.pk, rawval, L) + elif PyUnicode_Check(o): + if not self.encoding: + raise TypeError("Can't encode unicode string: no encoding is specified") + o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) + rawval = o + ret = msgpack_pack_raw(&self.pk, len(o)) + if ret == 0: + ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) + elif PyDict_CheckExact(o): + d = o + ret = msgpack_pack_map(&self.pk, len(d)) + if ret == 0: + for k, v in d.iteritems(): + ret = self._pack(k, nest_limit-1) + if ret != 0: break + ret = self._pack(v, nest_limit-1) + if ret != 0: break + elif PyDict_Check(o): + ret = msgpack_pack_map(&self.pk, len(o)) + if ret == 0: + for k, v in o.items(): + ret = self._pack(k, nest_limit-1) + if ret != 0: break + ret = self._pack(v, nest_limit-1) + if ret != 0: break + elif isinstance(o, ExtType): + # This should be before Tuple because ExtType is namedtuple. + longval = o[0] + rawval = o[1] + L = len(o[1]) + ret = msgpack_pack_ext(&self.pk, longval, L) ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyUnicode_Check(o): - if not self.encoding: - raise TypeError("Can't encode unicode string: no encoding is specified") - o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) - rawval = o - ret = msgpack_pack_raw(&self.pk, len(o)) - if ret == 0: - ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) - elif PyDict_CheckExact(o): - d = o - ret = msgpack_pack_map(&self.pk, len(d)) - if ret == 0: - for k, v in d.iteritems(): - ret = self._pack(k, nest_limit-1) - if ret != 0: break - ret = self._pack(v, nest_limit-1) - if ret != 0: break - elif PyDict_Check(o): - ret = msgpack_pack_map(&self.pk, len(o)) - if ret == 0: - for k, v in o.items(): - ret = self._pack(k, nest_limit-1) - if ret != 0: break - ret = self._pack(v, nest_limit-1) - if ret != 0: break - elif PyTuple_Check(o) or PyList_Check(o): - ret = msgpack_pack_array(&self.pk, len(o)) - if ret == 0: - for v in o: - ret = self._pack(v, nest_limit-1) - if ret != 0: break - elif self.handle_unknown_type(o): - # it means that obj was succesfully packed, so we are done - return 0 - elif self._default: - o = self._default(o) - ret = self._pack(o, nest_limit-1) - else: - raise TypeError("can't serialize %r" % (o,)) - return ret + elif PyTuple_Check(o) or PyList_Check(o): + ret = msgpack_pack_array(&self.pk, len(o)) + if ret == 0: + for v in o: + ret = self._pack(v, nest_limit-1) + if ret != 0: break + elif not default_used and self._default: + o = self._default(o) + default_used = 1 + continue + else: + raise TypeError("can't serialize %r" % (o,)) + return ret cpdef pack(self, object obj): cdef int ret @@ -207,9 +216,6 @@ cdef class Packer(object): self.pk.length = 0 return buf - def handle_unknown_type(self, obj): - return None - def pack_ext_type(self, typecode, data): msgpack_pack_ext(&self.pk, typecode, len(data)) msgpack_pack_raw_body(&self.pk, data, len(data)) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index b0e66db..d5aa46e 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -16,6 +16,7 @@ from msgpack.exceptions import ( UnpackValueError, ExtraData, ) +from msgpack import ExtType cdef extern from "unpack.h": @@ -24,7 +25,7 @@ cdef extern from "unpack.h": PyObject* object_hook bint has_pairs_hook # call object_hook with k-v pairs PyObject* list_hook - PyObject* ext_type_hook + PyObject* ext_hook char *encoding char *unicode_errors @@ -43,8 +44,8 @@ cdef extern from "unpack.h": object unpack_data(unpack_context* ctx) cdef inline init_ctx(unpack_context *ctx, - object object_hook, object object_pairs_hook, object list_hook, - object ext_type_hook, + object object_hook, object object_pairs_hook, + object list_hook, object ext_hook, bint use_list, char* encoding, char* unicode_errors): unpack_init(ctx) ctx.user.use_list = use_list @@ -71,10 +72,10 @@ cdef inline init_ctx(unpack_context *ctx, raise TypeError("list_hook must be a callable.") ctx.user.list_hook = list_hook - if ext_type_hook is not None: - if not PyCallable_Check(ext_type_hook): - raise TypeError("ext_type_hook must be a callable.") - ctx.user.ext_type_hook = ext_type_hook + if ext_hook is not None: + if not PyCallable_Check(ext_hook): + raise TypeError("ext_hook must be a callable.") + ctx.user.ext_hook = ext_hook ctx.user.encoding = encoding ctx.user.unicode_errors = unicode_errors @@ -84,8 +85,7 @@ def default_read_extended_type(typecode, data): def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=1, encoding=None, unicode_errors="strict", - object_pairs_hook=None, - ): + object_pairs_hook=None, ext_hook=ExtType): """ Unpack packed_bytes to object. Returns an unpacked object. @@ -114,8 +114,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, unicode_errors = unicode_errors.encode('ascii') cerr = PyBytes_AsString(unicode_errors) - init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, default_read_extended_type, - use_list, cenc, cerr) + init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook, + use_list, cenc, cerr) ret = unpack_construct(&ctx, buf, buf_len, &off) if ret == 1: obj = unpack_data(&ctx) @@ -220,7 +220,7 @@ cdef class Unpacker(object): def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1, object object_hook=None, object object_pairs_hook=None, object list_hook=None, str encoding=None, str unicode_errors='strict', int max_buffer_size=0, - ): + object ext_hook=ExtType): cdef char *cenc=NULL, *cerr=NULL self.file_like = file_like @@ -257,10 +257,8 @@ cdef class Unpacker(object): self.unicode_errors = unicode_errors cerr = PyBytes_AsString(self.unicode_errors) - ext_type_hook = self.read_extended_type - Py_INCREF(ext_type_hook) init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, - ext_type_hook, use_list, cenc, cerr) + ext_hook, use_list, cenc, cerr) def feed(self, object next_bytes): """Append `next_bytes` to internal buffer.""" @@ -370,24 +368,6 @@ cdef class Unpacker(object): """ return self._unpack(unpack_construct, write_bytes) - def unpack_one(self, object write_bytes=None): - """ - unpack one object - - If write_bytes is not None, it will be called with parts of the raw - message as it is unpacked. - - Raises `UnpackValueError` if there are no more bytes to unpack. - Raises ``ExtraData`` if there are still bytes left after the unpacking. - """ - try: - result = self.unpack() - except OutOfData: - raise UnpackValueError("Data is not enough") - if self.buf_head < self.buf_tail: - raise ExtraData(result, self.buf[self.buf_head:]) - return result - def skip(self, object write_bytes=None): """ read and ignore one object, returning None @@ -415,9 +395,6 @@ cdef class Unpacker(object): """ return self._unpack(read_map_header, write_bytes) - def read_extended_type(self, typecode, data): - return default_read_extended_type(typecode, data) - def __iter__(self): return self diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 0fe9514..8b91619 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -687,7 +687,7 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l) static inline int msgpack_pack_bin(msgpack_packer *x, size_t l) { if (!x->use_bin_type) { - return msgpack_pack_raw(x, l) + return msgpack_pack_raw(x, l); } if (l < 256) { unsigned char buf[2] = {0xc4, (unsigned char)l}; @@ -711,7 +711,6 @@ static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t /* * Ext */ - static inline int msgpack_pack_ext(msgpack_packer* x, int8_t typecode, size_t l) { if (l == 1) { diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 327a524..3c09747 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -24,7 +24,7 @@ typedef struct unpack_user { PyObject *object_hook; bool has_pairs_hook; PyObject *list_hook; - PyObject *ext_type_hook; + PyObject *ext_hook; const char *encoding; const char *unicode_errors; } unpack_user; @@ -241,12 +241,12 @@ static inline int unpack_callback_ext(unpack_user* u, const char* base, const ch { PyObject *py; int8_t typecode = (int8_t)*pos++; - if (!u->ext_type_hook) { - PyErr_SetString(PyExc_AssertionError, "u->ext_type_hook cannot be NULL"); + if (!u->ext_hook) { + PyErr_SetString(PyExc_AssertionError, "u->ext_hook cannot be NULL"); return -1; } - // lenght also includes the typecode, so the actual data is lenght-1 - py = PyEval_CallFunction(u->ext_type_hook, "(is#)", typecode, pos, lenght-1); + // length also includes the typecode, so the actual data is lenght-1 + py = PyEval_CallFunction(u->ext_hook, "(is#)", typecode, pos, lenght-1); if (!py) return -1; *o = py; diff --git a/test/test_extension.py b/test/test_extension.py index f2fa363..2f85ce3 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,5 +1,7 @@ +from __future__ import print_function import array import msgpack +from msgpack import ExtType def test_pack_ext_type(): @@ -11,54 +13,45 @@ def test_pack_ext_type(): assert p(b'AB') == b'\xd5\x42AB' # fixext 2 assert p(b'ABCD') == b'\xd6\x42ABCD' # fixext 4 assert p(b'ABCDEFGH') == b'\xd7\x42ABCDEFGH' # fixext 8 - assert p(b'A'*16) == b'\xd8\x42' + 'A'*16 # fixext 16 + assert p(b'A'*16) == b'\xd8\x42' + b'A'*16 # fixext 16 assert p(b'ABC') == b'\xc7\x03\x42ABC' # ext 8 assert p(b'A'*0x0123) == b'\xc8\x01\x23\x42' + b'A'*0x0123 # ext 16 assert p(b'A'*0x00012345) == b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345 # ext 32 -def test_unpack_extended_type(): - class MyUnpacker(msgpack.Unpacker): - def read_extended_type(self, typecode, data): - return (typecode, data) +def test_unpack_ext_type(): + def check(b, expected): + assert msgpack.unpackb(b) == expected - def u(s): - unpacker = MyUnpacker() - unpacker.feed(s) - return unpacker.unpack_one() - - assert u('\xd4\x42A') == (0x42, 'A') # fixext 1 - assert u('\xd5\x42AB') == (0x42, 'AB') # fixext 2 - assert u('\xd6\x42ABCD') == (0x42, 'ABCD') # fixext 4 - assert u('\xd7\x42ABCDEFGH') == (0x42, 'ABCDEFGH') # fixext 8 - assert u('\xd8\x42' + 'A'*16) == (0x42, 'A'*16) # fixext 16 - assert u('\xc7\x03\x42ABC') == (0x42, 'ABC') # ext 8 - assert (u('\xc8\x01\x23\x42' + 'A'*0x0123) == - (0x42, 'A'*0x0123)) # ext 16 - assert (u('\xc9\x00\x01\x23\x45\x42' + 'A'*0x00012345) == - (0x42, 'A'*0x00012345)) # ext 32 + check(b'\xd4\x42A', ExtType(0x42, b'A')) # fixext 1 + check(b'\xd5\x42AB', ExtType(0x42, b'AB')) # fixext 2 + check(b'\xd6\x42ABCD', ExtType(0x42, b'ABCD')) # fixext 4 + check(b'\xd7\x42ABCDEFGH', ExtType(0x42, b'ABCDEFGH')) # fixext 8 + check(b'\xd8\x42' + b'A'*16, ExtType(0x42, b'A'*16)) # fixext 16 + check(b'\xc7\x03\x42ABC', ExtType(0x42, b'ABC')) # ext 8 + check(b'\xc8\x01\x23\x42' + b'A'*0x0123, + ExtType(0x42, b'A'*0x0123)) # ext 16 + check(b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345, + ExtType(0x42, b'A'*0x00012345)) # ext 32 def test_extension_type(): - class MyPacker(msgpack.Packer): - def handle_unknown_type(self, obj): - if isinstance(obj, array.array): - typecode = 123 # application specific typecode - data = obj.tostring() - self.pack_ext_type(typecode, data) - return True + def default(obj): + print('default called', obj) + if isinstance(obj, array.array): + typecode = 123 # application specific typecode + data = obj.tostring() + return ExtType(typecode, data) + raise TypeError("Unknwon type object %r" % (obj,)) - class MyUnpacker(msgpack.Unpacker): - def read_extended_type(self, typecode, data): - assert typecode == 123 - obj = array.array('d') - obj.fromstring(data) - return obj + def ext_hook(code, data): + print('ext_hook called', code, data) + assert code == 123 + obj = array.array('d') + obj.fromstring(data) + return obj - obj = [42, 'hello', array.array('d', [1.1, 2.2, 3.3])] - packer = MyPacker() - unpacker = MyUnpacker(None) - s = packer.pack(obj) - unpacker.feed(s) - obj2 = unpacker.unpack_one() + obj = [42, b'hello', array.array('d', [1.1, 2.2, 3.3])] + s = msgpack.packb(obj, default=default) + obj2 = msgpack.unpackb(s, ext_hook=ext_hook) assert obj == obj2 diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index abc447a..af66b78 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -85,16 +85,3 @@ def test_readbytes(): assert unpacker.read_bytes(3) == b'oob' assert unpacker.unpack() == ord(b'a') assert unpacker.unpack() == ord(b'r') - -def test_unpack_one(): - unpacker = Unpacker() - unpacker.feed('\xda\x00\x03abc') - assert unpacker.unpack_one() == 'abc' - # - unpacker = Unpacker() - unpacker.feed('\xda\x00\x03abcd') - py.test.raises(ExtraData, "unpacker.unpack_one()") - # - unpacker = Unpacker() - unpacker.feed('\xda\x00\x03ab') - py.test.raises(UnpackValueError, "unpacker.unpack_one()") From 822cce823cfea8e9f7625598a125897718b4ab58 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 20 Oct 2013 22:59:27 +0900 Subject: [PATCH 0733/1172] Support unpacking new types. --- msgpack/unpack.h | 10 +++++++--- msgpack/unpack_define.h | 1 - msgpack/unpack_template.h | 15 +++++++++------ test/test_obj.py | 2 +- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 3c09747..c733b24 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -157,7 +157,7 @@ static inline int unpack_callback_array_item(unpack_user* u, unsigned int curren static inline int unpack_callback_array_end(unpack_user* u, msgpack_unpack_object* c) { if (u->list_hook) { - PyObject *new_c = PyEval_CallFunction(u->list_hook, "(O)", *c); + PyObject *new_c = PyObject_CallFunction(u->list_hook, "(O)", *c); if (!new_c) return -1; Py_DECREF(*c); @@ -203,7 +203,7 @@ static inline int unpack_callback_map_item(unpack_user* u, unsigned int current, static inline int unpack_callback_map_end(unpack_user* u, msgpack_unpack_object* c) { if (u->object_hook) { - PyObject *new_c = PyEval_CallFunction(u->object_hook, "(O)", *c); + PyObject *new_c = PyObject_CallFunction(u->object_hook, "(O)", *c); if (!new_c) return -1; @@ -246,7 +246,11 @@ static inline int unpack_callback_ext(unpack_user* u, const char* base, const ch return -1; } // length also includes the typecode, so the actual data is lenght-1 - py = PyEval_CallFunction(u->ext_hook, "(is#)", typecode, pos, lenght-1); +#if PY_MAJOR_VERSION == 2 + py = PyObject_CallFunction(u->ext_hook, "(is#)", typecode, pos, lenght-1); +#else + py = PyObject_CallFunction(u->ext_hook, "(iy#)", typecode, pos, lenght-1); +#endif if (!py) return -1; *o = py; diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 2ee92b5..0dd708d 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -93,4 +93,3 @@ typedef enum { #endif #endif /* msgpack/unpack_define.h */ - diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 1a709ec..7646896 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -178,15 +178,17 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l switch(*p) { case 0xc0: // nil push_simple_value(_nil); - //case 0xc1: // string - // again_terminal_trail(NEXT_CS(p), p+1); + //case 0xc1: // never used case 0xc2: // false push_simple_value(_false); case 0xc3: // true push_simple_value(_true); - //case 0xc4: - //case 0xc5: - //case 0xc6: + case 0xc4: // bin 8 + again_fixed_trail(NEXT_CS(p), 1); + case 0xc5: // bin 16 + again_fixed_trail(NEXT_CS(p), 2); + case 0xc6: // bin 32 + again_fixed_trail(NEXT_CS(p), 4); case 0xc7: // ext 8 again_fixed_trail(NEXT_CS(p), 1); case 0xc8: // ext 16 @@ -213,7 +215,8 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l _ext_zero); case 0xd8: // fixext 16 again_fixed_trail_if_zero(ACS_EXT_VALUE, 16+1, _ext_zero); - //case 0xd9: + case 0xd9: // str 8 + again_fixed_trail(NEXT_CS(p), 1); case 0xda: // raw 16 case 0xdb: // raw 32 case 0xdc: // array 16 diff --git a/test/test_obj.py b/test/test_obj.py index fbf610c..9083218 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -35,7 +35,7 @@ def test_only_one_obj_hook(): unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x) def test_bad_hook(): - with raises(ValueError): + with raises(TypeError): packed = packb([3, 1+2j], default=lambda o: o) unpacked = unpackb(packed, use_list=1) From 0d5c58bd517caddd6b62a8931f6833e2a3add283 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 20 Oct 2013 23:06:02 +0900 Subject: [PATCH 0734/1172] cleanup --- msgpack/unpack_template.h | 2 +- setup.py | 1 - test/test_sequnpack.py | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 7646896..d34eced 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -322,7 +322,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l case CS_MAP_32: /* FIXME security guard */ start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); - + default: goto _failed; } diff --git a/setup.py b/setup.py index 83ae79f..1055a61 100644 --- a/setup.py +++ b/setup.py @@ -92,7 +92,6 @@ if not hasattr(sys, 'pypy_version_info'): libraries=libraries, include_dirs=['.'], define_macros=macros, - extra_compile_args=['-O0'], )) del libraries, macros diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index af66b78..f541207 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,10 +1,9 @@ #!/usr/bin/env python # coding: utf-8 -import py import six from msgpack import Unpacker, BufferFull -from msgpack.exceptions import OutOfData, ExtraData, UnpackValueError +from msgpack.exceptions import OutOfData from pytest import raises From 84dc99c894be82b7a8c3708a3554888a6133b33b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 20 Oct 2013 23:27:32 +0900 Subject: [PATCH 0735/1172] Add ext_type example to README. --- README.rst | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 600d7f7..99fb923 100644 --- a/README.rst +++ b/README.rst @@ -143,10 +143,27 @@ key-value pairs. Extended types ^^^^^^^^^^^^^^^ -It is also possible to pack/unpack custom data types using the msgpack feature -of "extended types". For example, msgpack-pypy uses it to provide very fast serialization of int/float lists on top of PyPy (experimental for now): +It is also possible to pack/unpack custom data types using the msgpack 2.0 feature. -https://bitbucket.org/antocuni/msgpack-pypy/src/default/msgpack_pypy.py + >>> import msgpack + >>> import array + >>> def default(obj): + ... if isinstance(obj, array.array) and obj.typecode == 'd': + ... return msgpack.ExtType(42, obj.tostring()) + ... raise TypeError("Unknown type: %r" % (obj,)) + ... + >>> def ext_hook(code, data): + ... if code == 42: + ... a = array.array('d') + ... a.fromstring(data) + ... return a + ... return ExtType(code, data) + ... + >>> data = array.array('d', [1.2, 3.4]) + >>> packed = msgpack.packb(data, default=default) + >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook) + >>> data == unpacked + True Advanced unpacking control From cb789596787592f4ec6bf7dcc0c646e8976b3f16 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 21 Oct 2013 00:01:47 +0900 Subject: [PATCH 0736/1172] Update README. --- README.rst | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 99fb923..c51e518 100644 --- a/README.rst +++ b/README.rst @@ -3,8 +3,8 @@ MessagePack for Python ======================= :author: INADA Naoki -:version: 0.3.0 -:date: 2012-12-07 +:version: 0.4.0 +:date: 2013-10-21 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png :target: https://travis-ci.org/#!/msgpack/msgpack-python @@ -39,8 +39,40 @@ amd64. Windows SDK is recommanded way to build amd64 msgpack without any fee.) Without extension, using pure python implementation on CPython runs slowly. +Notes +----- + +Note for msgpack 2.0 support +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +msgpack 2.0 adds two types: *bin* and *ext*. + +*raw* was bytes or string type like Python 2's ``str``. +To distinguish string and bytes, msgpack 2.0 adds *bin*. +It is non-string binary like Python 3's ``bytes``. + +To use *bin* type for packing ``bytes``, pass ``use_bin_type=True`` to +packer argument. + + >>> import msgpack + >>> packed = msgpack.packb([b'spam', u'egg'], use_bin_type=True) + >>> msgpack.unpackb(packed, encoding='utf-8') + ['spam', u'egg'] + +You shoud use it carefully. When you use ``use_bin_type=True``, packed +binary can be unpacked by unpackers supporting msgpack-2.0. + +To use *ext* type, pass ``msgpack.ExtType`` object to packer. + + >>> import msgpack + >>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy')) + >>> msgpack.unpackb(packed) + ExtType(code=42, data='xyzzy') + +You can use it with ``default`` and ``ext_hook``. See below. + Note for msgpack 0.2.x users ----------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The msgpack 0.3 have some incompatible changes. From 37c2ad63af8a6e5cb6944f80d931fedbc6b49e7d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 21 Oct 2013 00:29:05 +0900 Subject: [PATCH 0737/1172] Add tests and bugfix. --- msgpack/pack_template.h | 3 ++- msgpack/unpack.h | 4 ++-- test/test_newspec.py | 23 +++++++++++++++++++++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 8b91619..2879bbd 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -705,7 +705,8 @@ static inline int msgpack_pack_bin(msgpack_packer *x, size_t l) static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t l) { - msgpack_pack_append_buffer(x, (const unsigned char*)b, l); + if (l > 0) msgpack_pack_append_buffer(x, (const unsigned char*)b, l); + return 0; } /* diff --git a/msgpack/unpack.h b/msgpack/unpack.h index c733b24..aced40b 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -157,7 +157,7 @@ static inline int unpack_callback_array_item(unpack_user* u, unsigned int curren static inline int unpack_callback_array_end(unpack_user* u, msgpack_unpack_object* c) { if (u->list_hook) { - PyObject *new_c = PyObject_CallFunction(u->list_hook, "(O)", *c); + PyObject *new_c = PyObject_CallFunctionObjArgs(u->list_hook, *c, NULL); if (!new_c) return -1; Py_DECREF(*c); @@ -203,7 +203,7 @@ static inline int unpack_callback_map_item(unpack_user* u, unsigned int current, static inline int unpack_callback_map_end(unpack_user* u, msgpack_unpack_object* c) { if (u->object_hook) { - PyObject *new_c = PyObject_CallFunction(u->object_hook, "(O)", *c); + PyObject *new_c = PyObject_CallFunctionObjArgs(u->object_hook, *c, NULL); if (!new_c) return -1; diff --git a/test/test_newspec.py b/test/test_newspec.py index 8bc2cfe..ab05029 100644 --- a/test/test_newspec.py +++ b/test/test_newspec.py @@ -1,6 +1,6 @@ # coding: utf-8 -from msgpack import packb, unpackb +from msgpack import packb, unpackb, ExtType def test_str8(): @@ -66,4 +66,23 @@ def test_bin32(): assert b[5:] == data assert unpackb(b) == data - +def test_ext(): + def check(ext, packed): + assert packb(ext) == packed + assert unpackb(packed) == ext + check(ExtType(0x42, b'Z'), b'\xd4\x42Z') # fixext 1 + check(ExtType(0x42, b'ZZ'), b'\xd5\x42ZZ') # fixext 2 + check(ExtType(0x42, b'Z'*4), b'\xd6\x42' + b'Z'*4) # fixext 4 + check(ExtType(0x42, b'Z'*8), b'\xd7\x42' + b'Z'*8) # fixext 8 + check(ExtType(0x42, b'Z'*16), b'\xd8\x42' + b'Z'*16) # fixext 16 + # ext 8 + check(ExtType(0x42, b''), b'\xc7\x00\x42') + check(ExtType(0x42, b'Z'*255), b'\xc7\xff\x42' + b'Z'*255) + # ext 16 + check(ExtType(0x42, b'Z'*256), b'\xc8\x01\x00\x42' + b'Z'*256) + check(ExtType(0x42, b'Z'*0xffff), b'\xc8\xff\xff\x42' + b'Z'*0xffff) + # ext 32 + check(ExtType(0x42, b'Z'*0x10000), b'\xc9\x00\x01\x00\x00\x42' + b'Z'*0x10000) + # needs large memory + #check(ExtType(0x42, b'Z'*0xffffffff), + # b'\xc9\xff\xff\xff\xff\x42' + b'Z'*0xffffffff) From e3fee4db5fbf1ead4a98fff6c8843574480c3c2a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 21 Oct 2013 00:59:22 +0900 Subject: [PATCH 0738/1172] fallback: support packing ExtType --- msgpack/__init__.py | 26 ++++++- msgpack/_packer.pyx | 5 +- msgpack/fallback.py | 181 +++++++++++++++++++++++++------------------- 3 files changed, 131 insertions(+), 81 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index a7b47b1..56a0b36 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -2,9 +2,31 @@ from msgpack._version import version from msgpack.exceptions import * -from collections import namedtuple -ExtType = namedtuple('ExtType', 'code data') +class ExtType(object): + __slots__ = ('code', 'data') + + def __init__(self, code, data): + if not isinstance(code, int): + raise TypeError("code must be int") + if not isinstance(data, bytes): + raise TypeError("data must be bytes") + if not 0 <= code <= 127: + raise ValueError("code must be 0~127") + self.code = code + self.data = data + + def __eq__(self, other): + if not isinstance(other, ExtType): + return NotImplemented + return self.code == other.code and self.data == other.data + + def __hash__(self): + return self.code ^ hash(self.data) + + def __repr__(self): + return "msgpack.ExtType(%r, %r)" % (self.code, self.data) + import os if os.environ.get('MSGPACK_PUREPYTHON'): diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index f63667c..f2d058e 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -37,7 +37,6 @@ cdef extern from "pack.h": cdef int DEFAULT_RECURSE_LIMIT=511 - cdef class Packer(object): """ MessagePack Packer @@ -185,8 +184,8 @@ cdef class Packer(object): if ret != 0: break elif isinstance(o, ExtType): # This should be before Tuple because ExtType is namedtuple. - longval = o[0] - rawval = o[1] + longval = o.code + rawval = o.data L = len(o[1]) ret = msgpack_pack_ext(&self.pk, longval, L) ret = msgpack_pack_raw_body(&self.pk, rawval, L) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 0b29700..bf5b1c2 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -506,82 +506,111 @@ class Packer(object): self._default = default def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance): - if nest_limit < 0: - raise PackValueError("recursion limit exceeded") - if obj is None: - return self._buffer.write(b"\xc0") - if isinstance(obj, bool): - if obj: - return self._buffer.write(b"\xc3") - return self._buffer.write(b"\xc2") - if isinstance(obj, int_types): - if 0 <= obj < 0x80: - return self._buffer.write(struct.pack("B", obj)) - if -0x20 <= obj < 0: - return self._buffer.write(struct.pack("b", obj)) - if 0x80 <= obj <= 0xff: - return self._buffer.write(struct.pack("BB", 0xcc, obj)) - if -0x80 <= obj < 0: - return self._buffer.write(struct.pack(">Bb", 0xd0, obj)) - if 0xff < obj <= 0xffff: - return self._buffer.write(struct.pack(">BH", 0xcd, obj)) - if -0x8000 <= obj < -0x80: - return self._buffer.write(struct.pack(">Bh", 0xd1, obj)) - if 0xffff < obj <= 0xffffffff: - return self._buffer.write(struct.pack(">BI", 0xce, obj)) - if -0x80000000 <= obj < -0x8000: - return self._buffer.write(struct.pack(">Bi", 0xd2, obj)) - if 0xffffffff < obj <= 0xffffffffffffffff: - return self._buffer.write(struct.pack(">BQ", 0xcf, obj)) - if -0x8000000000000000 <= obj < -0x80000000: - return self._buffer.write(struct.pack(">Bq", 0xd3, obj)) - raise PackValueError("Integer value out of range") - if self._use_bin_type and isinstance(obj, bytes): - n = len(obj) - if n <= 0xff: - self._buffer.write(struct.pack('>BB', 0xc4, n)) - elif n <= 0xffff: - self._buffer.write(struct.pack(">BH", 0xc5, n)) - elif n <= 0xffffffff: - self._buffer.write(struct.pack(">BI", 0xc6, n)) - else: - raise PackValueError("Bytes is too large") - return self._buffer.write(obj) - if isinstance(obj, (Unicode, bytes)): - if isinstance(obj, Unicode): - if self._encoding is None: - raise TypeError( - "Can't encode unicode string: " - "no encoding is specified") - obj = obj.encode(self._encoding, self._unicode_errors) - n = len(obj) - if n <= 0x1f: - self._buffer.write(struct.pack('B', 0xa0 + n)) - elif self._use_bin_type and n <= 0xff: - self._buffer.write(struct.pack('>BB', 0xd9, n)) - elif n <= 0xffff: - self._buffer.write(struct.pack(">BH", 0xda, n)) - elif n <= 0xffffffff: - self._buffer.write(struct.pack(">BI", 0xdb, n)) - else: - raise PackValueError("String is too large") - return self._buffer.write(obj) - if isinstance(obj, float): - if self._use_float: - return self._buffer.write(struct.pack(">Bf", 0xca, obj)) - return self._buffer.write(struct.pack(">Bd", 0xcb, obj)) - if isinstance(obj, (list, tuple)): - n = len(obj) - self._fb_pack_array_header(n) - for i in xrange(n): - self._pack(obj[i], nest_limit - 1) - return - if isinstance(obj, dict): - return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj), - nest_limit - 1) - if self._default is not None: - return self._pack(self._default(obj), nest_limit - 1) - raise TypeError("Cannot serialize %r" % obj) + default_used = False + while True: + if nest_limit < 0: + raise PackValueError("recursion limit exceeded") + if obj is None: + return self._buffer.write(b"\xc0") + if isinstance(obj, bool): + if obj: + return self._buffer.write(b"\xc3") + return self._buffer.write(b"\xc2") + if isinstance(obj, int_types): + if 0 <= obj < 0x80: + return self._buffer.write(struct.pack("B", obj)) + if -0x20 <= obj < 0: + return self._buffer.write(struct.pack("b", obj)) + if 0x80 <= obj <= 0xff: + return self._buffer.write(struct.pack("BB", 0xcc, obj)) + if -0x80 <= obj < 0: + return self._buffer.write(struct.pack(">Bb", 0xd0, obj)) + if 0xff < obj <= 0xffff: + return self._buffer.write(struct.pack(">BH", 0xcd, obj)) + if -0x8000 <= obj < -0x80: + return self._buffer.write(struct.pack(">Bh", 0xd1, obj)) + if 0xffff < obj <= 0xffffffff: + return self._buffer.write(struct.pack(">BI", 0xce, obj)) + if -0x80000000 <= obj < -0x8000: + return self._buffer.write(struct.pack(">Bi", 0xd2, obj)) + if 0xffffffff < obj <= 0xffffffffffffffff: + return self._buffer.write(struct.pack(">BQ", 0xcf, obj)) + if -0x8000000000000000 <= obj < -0x80000000: + return self._buffer.write(struct.pack(">Bq", 0xd3, obj)) + raise PackValueError("Integer value out of range") + if self._use_bin_type and isinstance(obj, bytes): + n = len(obj) + if n <= 0xff: + self._buffer.write(struct.pack('>BB', 0xc4, n)) + elif n <= 0xffff: + self._buffer.write(struct.pack(">BH", 0xc5, n)) + elif n <= 0xffffffff: + self._buffer.write(struct.pack(">BI", 0xc6, n)) + else: + raise PackValueError("Bytes is too large") + return self._buffer.write(obj) + if isinstance(obj, (Unicode, bytes)): + if isinstance(obj, Unicode): + if self._encoding is None: + raise TypeError( + "Can't encode unicode string: " + "no encoding is specified") + obj = obj.encode(self._encoding, self._unicode_errors) + n = len(obj) + if n <= 0x1f: + self._buffer.write(struct.pack('B', 0xa0 + n)) + elif self._use_bin_type and n <= 0xff: + self._buffer.write(struct.pack('>BB', 0xd9, n)) + elif n <= 0xffff: + self._buffer.write(struct.pack(">BH", 0xda, n)) + elif n <= 0xffffffff: + self._buffer.write(struct.pack(">BI", 0xdb, n)) + else: + raise PackValueError("String is too large") + return self._buffer.write(obj) + if isinstance(obj, float): + if self._use_float: + return self._buffer.write(struct.pack(">Bf", 0xca, obj)) + return self._buffer.write(struct.pack(">Bd", 0xcb, obj)) + if isinstance(obj, ExtType): + code = obj.code + data = obj.data + assert isinstance(code, int) + assert isinstance(data, bytes) + L = len(data) + if L == 1: + self._buffer.write(b'\xd4') + elif L == 2: + self._buffer.write(b'\xd5') + elif L == 4: + self._buffer.write(b'\xd6') + elif L == 8: + self._buffer.write(b'\xd7') + elif L == 16: + self._buffer.write(b'\xd8') + elif L <= 0xff: + self._buffer.write(struct.pack(">BB", 0xc7, L)) + elif L <= 0xffff: + self._buffer.write(struct.pack(">BH", 0xc8, L)) + else: + self._buffer.write(struct.pack(">BI", 0xc9, L)) + self._buffer.write(struct.pack("b", code)) + self._buffer.write(data) + return + if isinstance(obj, (list, tuple)): + n = len(obj) + self._fb_pack_array_header(n) + for i in xrange(n): + self._pack(obj[i], nest_limit - 1) + return + if isinstance(obj, dict): + return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj), + nest_limit - 1) + if not default_used and self._default is not None: + obj = self._default(obj) + default_used = 1 + continue + raise TypeError("Cannot serialize %r" % obj) def pack(self, obj): self._pack(obj) From d84a403bc0bbbb36c4a5833e00269eef6c4a91ae Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 21 Oct 2013 01:12:57 +0900 Subject: [PATCH 0739/1172] fix bugs. --- msgpack/__init__.py | 20 ++++---------------- msgpack/_packer.pyx | 2 +- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 56a0b36..a958025 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -2,30 +2,18 @@ from msgpack._version import version from msgpack.exceptions import * +from collections import namedtuple -class ExtType(object): - __slots__ = ('code', 'data') - def __init__(self, code, data): +class ExtType(namedtuple('ExtType', 'code data')): + def __new__(cls, code, data): if not isinstance(code, int): raise TypeError("code must be int") if not isinstance(data, bytes): raise TypeError("data must be bytes") if not 0 <= code <= 127: raise ValueError("code must be 0~127") - self.code = code - self.data = data - - def __eq__(self, other): - if not isinstance(other, ExtType): - return NotImplemented - return self.code == other.code and self.data == other.data - - def __hash__(self): - return self.code ^ hash(self.data) - - def __repr__(self): - return "msgpack.ExtType(%r, %r)" % (self.code, self.data) + return super(ExtType, cls).__new__(cls, code, data) import os diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index f2d058e..f261f08 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -186,7 +186,7 @@ cdef class Packer(object): # This should be before Tuple because ExtType is namedtuple. longval = o.code rawval = o.data - L = len(o[1]) + L = len(o.data) ret = msgpack_pack_ext(&self.pk, longval, L) ret = msgpack_pack_raw_body(&self.pk, rawval, L) elif PyTuple_Check(o) or PyList_Check(o): From 1d0096b998abca1276b88ae6960316f8cff273b9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 21 Oct 2013 01:20:13 +0900 Subject: [PATCH 0740/1172] 0.4.0 --- msgpack/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index 5999ede..e7a935c 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 3, 0) +version = (0, 4, 0) From f31a4403a1715ea5b48a12e1aa58df0e74b7e85f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 21 Oct 2013 01:47:54 +0900 Subject: [PATCH 0741/1172] Document update. --- Makefile | 3 +++ docs/api.rst | 2 ++ docs/conf.py | 2 +- msgpack/__init__.py | 1 + 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 71be6af..2e53d08 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,9 @@ doc-serve: all doc: cd docs && make zip +upload-doc: + python setup.py upload_docs --upload-dir docs/_build/html + cython: cython --cplus msgpack/*.pyx diff --git a/docs/api.rst b/docs/api.rst index 50a84c4..841c134 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -25,6 +25,8 @@ API reference .. autoclass:: Unpacker :members: +.. autoclass:: ExtType + exceptions ----------- diff --git a/docs/conf.py b/docs/conf.py index fba09b7..0f19fcc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -49,7 +49,7 @@ copyright = u'2013, INADA Naoki' # # The short X.Y version. # The full version, including alpha/beta/rc tags. -version = release = '0.3' +version = release = '0.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/msgpack/__init__.py b/msgpack/__init__.py index a958025..6c5ae53 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -6,6 +6,7 @@ from collections import namedtuple class ExtType(namedtuple('ExtType', 'code data')): + """ExtType represents ext type in msgpack.""" def __new__(cls, code, data): if not isinstance(code, int): raise TypeError("code must be int") From d5436c28197f3ccdd473e1c54838133574948dab Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 21 Oct 2013 02:33:58 +0900 Subject: [PATCH 0742/1172] Update ChangeLog --- ChangeLog.rst | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index fe64ff8..c5cff15 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,20 @@ +0.4.0 +===== +:release date: 2013-10-21 + +Inconpatible Changes +-------------------- + +* Raises TypeError instead of ValueError when packer receives unsupported type. + +Changes +------- + +* Support New msgpack spec. + + 0.3.0 ===== -:release date: in development Inconpatible Changes -------------------- From 48ca2d700d019bfec9abe3242a8eb3073c803951 Mon Sep 17 00:00:00 2001 From: Sergey Zhuravlev Date: Sun, 15 Dec 2013 16:22:39 +0000 Subject: [PATCH 0743/1172] Added support of bytearrays to msgpack/fallback.py --- msgpack/fallback.py | 2 ++ test/test_buffer.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index bf5b1c2..3ef1341 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -193,6 +193,8 @@ class Unpacker(object): def feed(self, next_bytes): if isinstance(next_bytes, array.array): next_bytes = next_bytes.tostring() + elif isinstance(next_bytes, bytearray): + next_bytes = str(next_bytes) assert self._fb_feeding if self._fb_buf_n + len(next_bytes) > self._max_buffer_size: raise BufferFull diff --git a/test/test_buffer.py b/test/test_buffer.py index 04cc02d..5ae87ac 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -11,3 +11,10 @@ def test_unpack_buffer(): obj = unpackb(buf, use_list=1) assert [b'foo', b'bar'] == obj + +def test_unpack_bytearray(): + buf = bytearray(packb(('foo', 'bar'))) + obj = unpackb(buf, use_list=1) + assert [b'foo', b'bar'] == obj + assert all(type(s)==str for s in obj) + From 77046b839da20feb528d303c8858ab6f34013ce7 Mon Sep 17 00:00:00 2001 From: Wouter Bolsterlee Date: Tue, 11 Feb 2014 20:34:23 +0100 Subject: [PATCH 0744/1172] Always raise TypeError for wrong argument types The code that checks whether hooks are callable() (and some other type checks) should always raise TypeError on failure. Before this change, both ValueError and TypeError were used in an inconsistent way (C extension and Python implementation were not the same). --- msgpack/_unpacker.pyx | 4 ++-- msgpack/fallback.py | 14 +++++++------- test/test_obj.py | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index d5aa46e..18592f4 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -52,7 +52,7 @@ cdef inline init_ctx(unpack_context *ctx, ctx.user.object_hook = ctx.user.list_hook = NULL if object_hook is not None and object_pairs_hook is not None: - raise ValueError("object_pairs_hook and object_hook are mutually exclusive.") + raise TypeError("object_pairs_hook and object_hook are mutually exclusive.") if object_hook is not None: if not PyCallable_Check(object_hook): @@ -227,7 +227,7 @@ cdef class Unpacker(object): if file_like: self.file_like_read = file_like.read if not PyCallable_Check(self.file_like_read): - raise ValueError("`file_like.read` must be a callable.") + raise TypeError("`file_like.read` must be a callable.") if not max_buffer_size: max_buffer_size = INT_MAX if read_size > max_buffer_size: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index bf5b1c2..cf61023 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -159,7 +159,7 @@ class Unpacker(object): self._fb_feeding = True else: if not callable(file_like.read): - raise ValueError("`file_like.read` must be callable") + raise TypeError("`file_like.read` must be callable") self.file_like = file_like self._fb_feeding = False self._fb_buffers = [] @@ -179,16 +179,16 @@ class Unpacker(object): self._ext_hook = ext_hook if list_hook is not None and not callable(list_hook): - raise ValueError('`list_hook` is not callable') + raise TypeError('`list_hook` is not callable') if object_hook is not None and not callable(object_hook): - raise ValueError('`object_hook` is not callable') + raise TypeError('`object_hook` is not callable') if object_pairs_hook is not None and not callable(object_pairs_hook): - raise ValueError('`object_pairs_hook` is not callable') + raise TypeError('`object_pairs_hook` is not callable') if object_hook is not None and object_pairs_hook is not None: - raise ValueError("object_pairs_hook and object_hook are mutually " - "exclusive") + raise TypeError("object_pairs_hook and object_hook are mutually " + "exclusive") if not callable(ext_hook): - raise ValueError("`ext_hook` is not callable") + raise TypeError("`ext_hook` is not callable") def feed(self, next_bytes): if isinstance(next_bytes, array.array): diff --git a/test/test_obj.py b/test/test_obj.py index 9083218..390c1b6 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -31,7 +31,7 @@ def test_decode_pairs_hook(): assert unpacked[1] == prod_sum def test_only_one_obj_hook(): - with raises(ValueError): + with raises(TypeError): unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x) def test_bad_hook(): From dd65341e0d3a0df843841796050f98ab1520e623 Mon Sep 17 00:00:00 2001 From: Wouter Bolsterlee Date: Tue, 11 Feb 2014 21:05:15 +0100 Subject: [PATCH 0745/1172] Cosmetic changes to fix Cython warnings Put declarations on separate line to avoid warnings like this: Non-trivial type declarators in shared declaration (e.g. mix of pointers and values). Each pointer declaration should be on its own line. --- msgpack/_unpacker.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index d5aa46e..a359507 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -221,7 +221,8 @@ cdef class Unpacker(object): object object_hook=None, object object_pairs_hook=None, object list_hook=None, str encoding=None, str unicode_errors='strict', int max_buffer_size=0, object ext_hook=ExtType): - cdef char *cenc=NULL, *cerr=NULL + cdef char *cenc=NULL, + cdef char *cerr=NULL self.file_like = file_like if file_like: From d2fc8010342512378e01322f8871c10a5974af4f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 13 Feb 2014 01:57:34 +0900 Subject: [PATCH 0746/1172] Fix warning on 64bit environment. --- msgpack/unpack.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index aced40b..27e3b62 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -56,9 +56,12 @@ static inline int unpack_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpac static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { PyObject *p; +#if UINT32_MAX > LONG_MAX if (d > LONG_MAX) { p = PyLong_FromUnsignedLong((unsigned long)d); - } else { + } else +#endif + { p = PyInt_FromLong((long)d); } if (!p) From 9d61f243878eeabd2042bb16fe22d4325e441da6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 13 Feb 2014 03:10:51 +0900 Subject: [PATCH 0747/1172] Feed data from file before _unpack() --- msgpack/_unpacker.pyx | 10 ++++++++++ test/test_unpack_file.py | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 test/test_unpack_file.py diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 732adef..16aca5c 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -327,8 +327,18 @@ cdef class Unpacker(object): cdef int ret cdef object obj cdef size_t prev_head + + if self.buf_head >= self.buf_tail and self.file_like is not None: + self.read_from_file() + while 1: prev_head = self.buf_head + if prev_head >= self.buf_tail: + if iter: + raise StopIteration("No more data to unpack.") + else: + raise OutOfData("No more data to unpack.") + ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) if write_bytes is not None: write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head)) diff --git a/test/test_unpack_file.py b/test/test_unpack_file.py new file mode 100644 index 0000000..1563008 --- /dev/null +++ b/test/test_unpack_file.py @@ -0,0 +1,19 @@ +from io import BytesIO +from msgpack import Unpacker, packb, OutOfData +from pytest import raises + + +def test_unpack_array_header_from_file(): + f = BytesIO(packb([1,2,3,4])) + unpacker = Unpacker(f) + assert unpacker.read_array_header() == 4 + assert unpacker.unpack() == 1 + assert unpacker.unpack() == 2 + assert unpacker.unpack() == 3 + assert unpacker.unpack() == 4 + with raises(OutOfData): + unpacker.unpack() + + +if __name__ == '__main__': + test_unpack_array_header_from_file() From 11a3b1561a07b28a37ac22637f636c4c5ac24c23 Mon Sep 17 00:00:00 2001 From: Sergey Zhuravlev Date: Wed, 12 Feb 2014 23:09:23 +0400 Subject: [PATCH 0748/1172] fixed support of python3 --- msgpack/fallback.py | 2 +- test/test_buffer.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 3ef1341..b673222 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -194,7 +194,7 @@ class Unpacker(object): if isinstance(next_bytes, array.array): next_bytes = next_bytes.tostring() elif isinstance(next_bytes, bytearray): - next_bytes = str(next_bytes) + next_bytes = (bytes if PY3 else str)(next_bytes) assert self._fb_feeding if self._fb_buf_n + len(next_bytes) > self._max_buffer_size: raise BufferFull diff --git a/test/test_buffer.py b/test/test_buffer.py index 5ae87ac..7b85e93 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -2,6 +2,7 @@ # coding: utf-8 from msgpack import packb, unpackb +import sys def test_unpack_buffer(): @@ -16,5 +17,7 @@ def test_unpack_bytearray(): buf = bytearray(packb(('foo', 'bar'))) obj = unpackb(buf, use_list=1) assert [b'foo', b'bar'] == obj - assert all(type(s)==str for s in obj) + expected_type = bytes if sys.version_info[0] == 3 else str + assert all(type(s)==expected_type for s in obj) + From 38cf835c95273dc20dd6cb3376c11a88b72c7fa9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 13 Feb 2014 09:40:12 +0900 Subject: [PATCH 0749/1172] Rename --- test/test_unpack.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 test/test_unpack.py diff --git a/test/test_unpack.py b/test/test_unpack.py new file mode 100644 index 0000000..1563008 --- /dev/null +++ b/test/test_unpack.py @@ -0,0 +1,19 @@ +from io import BytesIO +from msgpack import Unpacker, packb, OutOfData +from pytest import raises + + +def test_unpack_array_header_from_file(): + f = BytesIO(packb([1,2,3,4])) + unpacker = Unpacker(f) + assert unpacker.read_array_header() == 4 + assert unpacker.unpack() == 1 + assert unpacker.unpack() == 2 + assert unpacker.unpack() == 3 + assert unpacker.unpack() == 4 + with raises(OutOfData): + unpacker.unpack() + + +if __name__ == '__main__': + test_unpack_array_header_from_file() From 0cab6092e456ffa04834233ffb01acb48d0869c3 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 13 Feb 2014 09:55:17 +0900 Subject: [PATCH 0750/1172] Add refcount check. --- test/test_unpack.py | 30 +++++++++++++++++++++++++++++- test/test_unpack_file.py | 19 ------------------- 2 files changed, 29 insertions(+), 20 deletions(-) delete mode 100644 test/test_unpack_file.py diff --git a/test/test_unpack.py b/test/test_unpack.py index 1563008..d6ca435 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -1,6 +1,7 @@ from io import BytesIO +import sys from msgpack import Unpacker, packb, OutOfData -from pytest import raises +from pytest import raises, mark def test_unpack_array_header_from_file(): @@ -15,5 +16,32 @@ def test_unpack_array_header_from_file(): unpacker.unpack() +@mark.skipif(not hasattr(sys, 'getrefcount'), + reason='sys.getrefcount() is needed to pass this test') +def test_unpacker_hook_refcnt(): + result = [] + + def hook(x): + result.append(x) + return x + + basecnt = sys.getrefcount(hook) + + up = Unpacker(object_pairs_hook=hook, list_hook=hook) + + assert sys.getrefcount(hook) >= basecnt + 2 + + up.feed(packb([{}])) + up.feed(packb([{}])) + assert up.unpack() == [{}] + assert up.unpack() == [{}] + assert result == [[{}], [{}]] + + del up + + assert sys.getrefcount(hook) == basecnt + + if __name__ == '__main__': test_unpack_array_header_from_file() + test_unpacker_hook_refcnt() diff --git a/test/test_unpack_file.py b/test/test_unpack_file.py deleted file mode 100644 index 1563008..0000000 --- a/test/test_unpack_file.py +++ /dev/null @@ -1,19 +0,0 @@ -from io import BytesIO -from msgpack import Unpacker, packb, OutOfData -from pytest import raises - - -def test_unpack_array_header_from_file(): - f = BytesIO(packb([1,2,3,4])) - unpacker = Unpacker(f) - assert unpacker.read_array_header() == 4 - assert unpacker.unpack() == 1 - assert unpacker.unpack() == 2 - assert unpacker.unpack() == 3 - assert unpacker.unpack() == 4 - with raises(OutOfData): - unpacker.unpack() - - -if __name__ == '__main__': - test_unpack_array_header_from_file() From cf63f19211797261b117227f23066952efebca29 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 13 Feb 2014 09:57:51 +0900 Subject: [PATCH 0751/1172] Fix test --- test/test_unpack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_unpack.py b/test/test_unpack.py index d6ca435..544cebf 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -27,7 +27,7 @@ def test_unpacker_hook_refcnt(): basecnt = sys.getrefcount(hook) - up = Unpacker(object_pairs_hook=hook, list_hook=hook) + up = Unpacker(object_hook=hook, list_hook=hook) assert sys.getrefcount(hook) >= basecnt + 2 @@ -35,7 +35,7 @@ def test_unpacker_hook_refcnt(): up.feed(packb([{}])) assert up.unpack() == [{}] assert up.unpack() == [{}] - assert result == [[{}], [{}]] + assert result == [{}, [{}], {}, [{}]] del up From 6d80569b9b7ec6f3756ecc91928e4ce127eb7a4b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 13 Feb 2014 09:58:38 +0900 Subject: [PATCH 0752/1172] Unpacker: maintain refcnt (fix #67). --- msgpack/_unpacker.pyx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 16aca5c..0df6ab3 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -206,7 +206,8 @@ cdef class Unpacker(object): cdef object file_like cdef object file_like_read cdef Py_ssize_t read_size - cdef object object_hook + # To maintain refcnt. + cdef object object_hook, object_pairs_hook, list_hook cdef object encoding, unicode_errors cdef size_t max_buffer_size @@ -224,6 +225,10 @@ cdef class Unpacker(object): cdef char *cenc=NULL, cdef char *cerr=NULL + self.object_hook = object_hook + self.object_pairs_hook = object_pairs_hook + self.list_hook = list_hook + self.file_like = file_like if file_like: self.file_like_read = file_like.read From a5368f62e24dd335bc7bb72980953ca707532693 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 14 Feb 2014 09:01:00 +0900 Subject: [PATCH 0753/1172] travis: Simplify .travis.yml and add PyPy env. --- .travis.yml | 10 ++-------- tox.ini | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index e536fdc..7cf7f58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,14 +2,8 @@ language: python python: - 2.7 -env: - - PIP_USE_MIRRORS=true - install: - - sudo apt-get update -qq - - sudo apt-get install -q python3.3-dev - - pip install --use-mirrors tox cython - - cython --cplus msgpack/_packer.pyx - - cython --cplus msgpack/_unpacker.pyx + - pip install tox cython + - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx script: "tox && MSGPACK_PUREPYTHON=x tox" diff --git a/tox.ini b/tox.ini index 1951352..2c5c596 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,py32,py33 +envlist = py26,py27,py32,py33,pypy [testenv] deps= From 213f7888c3edbe828533ec08e7a07bac3a98b482 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 14 Feb 2014 09:26:46 +0900 Subject: [PATCH 0754/1172] Remove too strict type check from test. --- test/test_pack.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_pack.py b/test/test_pack.py index 9dc5ada..0d3ed6d 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -12,7 +12,6 @@ from io import BytesIO def check(data, use_list=False): re = unpackb(packb(data), use_list=use_list) assert re == data - assert type(re) == type(data) def testPack(): test_data = [ From dee2d87d413eaf9cb38f11c72df10e82a73f452f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 15 Feb 2014 22:16:01 +0900 Subject: [PATCH 0755/1172] six.BytesIO => io.BytesIO --- test/test_pack.py | 6 +++--- test/test_sequnpack.py | 4 ++-- test/test_unpack_raw.py | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/test_pack.py b/test/test_pack.py index 0d3ed6d..2da7089 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -89,7 +89,7 @@ def testPackFloat(): assert packb(1.0, use_single_float=False) == b'\xcb' + struct.pack('>d', 1.0) def testArraySize(sizes=[0, 5, 50, 1000]): - bio = six.BytesIO() + bio = BytesIO() packer = Packer() for size in sizes: bio.write(packer.pack_array_header(size)) @@ -108,7 +108,7 @@ def test_manualreset(sizes=[0, 5, 50, 1000]): for i in range(size): packer.pack(i) - bio = six.BytesIO(packer.bytes()) + bio = BytesIO(packer.bytes()) unpacker = Unpacker(bio, use_list=1) for size in sizes: assert unpacker.unpack() == list(range(size)) @@ -117,7 +117,7 @@ def test_manualreset(sizes=[0, 5, 50, 1000]): assert packer.bytes() == b'' def testMapSize(sizes=[0, 5, 50, 1000]): - bio = six.BytesIO() + bio = BytesIO() packer = Packer() for size in sizes: bio.write(packer.pack_map_header(size)) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index f541207..5d37698 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # coding: utf-8 -import six +import io from msgpack import Unpacker, BufferFull from msgpack.exceptions import OutOfData from pytest import raises @@ -79,7 +79,7 @@ def test_readbytes(): assert unpacker.unpack() == ord(b'r') # Test buffer refill - unpacker = Unpacker(six.BytesIO(b'foobar'), read_size=3) + unpacker = Unpacker(io.BytesIO(b'foobar'), read_size=3) assert unpacker.unpack() == ord(b'f') assert unpacker.read_bytes(3) == b'oob' assert unpacker.unpack() == ord(b'a') diff --git a/test/test_unpack_raw.py b/test/test_unpack_raw.py index 9f3784c..7002601 100644 --- a/test/test_unpack_raw.py +++ b/test/test_unpack_raw.py @@ -1,19 +1,19 @@ """Tests for cases where the user seeks to obtain packed msgpack objects""" -import six +import io from msgpack import Unpacker, packb def test_write_bytes(): unpacker = Unpacker() unpacker.feed(b'abc') - f = six.BytesIO() + f = io.BytesIO() assert unpacker.unpack(f.write) == ord('a') assert f.getvalue() == b'a' - f = six.BytesIO() + f = io.BytesIO() assert unpacker.skip(f.write) is None assert f.getvalue() == b'b' - f = six.BytesIO() + f = io.BytesIO() assert unpacker.skip() is None assert f.getvalue() == b'' @@ -21,9 +21,9 @@ def test_write_bytes(): def test_write_bytes_multi_buffer(): long_val = (5) * 100 expected = packb(long_val) - unpacker = Unpacker(six.BytesIO(expected), read_size=3, max_buffer_size=3) + unpacker = Unpacker(io.BytesIO(expected), read_size=3, max_buffer_size=3) - f = six.BytesIO() + f = io.BytesIO() unpacked = unpacker.unpack(f.write) assert unpacked == long_val assert f.getvalue() == expected From 63eab502dff62bf7adbc25c75e8bc5bffa6ee234 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 15 Feb 2014 22:20:57 +0900 Subject: [PATCH 0756/1172] Remove six.b() --- test/test_seq.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/test_seq.py b/test/test_seq.py index af719b0..fed9ff4 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -1,16 +1,15 @@ #!/usr/bin/env python # coding: utf-8 -import six import io import msgpack -binarydata = [chr(i) for i in range(256)] -binarydata = six.b("".join(binarydata)) + +binarydata = bytes(bytearray(range(256))) def gen_binary_data(idx): - data = binarydata[:idx % 300] - return data + return binarydata[:idx % 300] + def test_exceeding_unpacker_read_size(): dumpf = io.BytesIO() From 0c22e775c9ea71242eec237e2e032aca749e909c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 15 Feb 2014 22:36:52 +0900 Subject: [PATCH 0757/1172] Remove six completely. --- test/test_pack.py | 30 ++++++++++++++---------------- tox.ini | 1 - 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/test/test_pack.py b/test/test_pack.py index 2da7089..762ccf5 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # coding: utf-8 +from __future__ import absolute_import, division, print_function, unicode_literals -import six import struct from pytest import raises, xfail @@ -28,24 +28,22 @@ def testPack(): check(td) def testPackUnicode(): - test_data = [ - six.u(""), six.u("abcd"), [six.u("defgh")], six.u("РуÑÑкий текÑÑ‚"), - ] + test_data = ["", "abcd", ["defgh"], "РуÑÑкий текÑÑ‚"] for td in test_data: re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8') assert re == td packer = Packer(encoding='utf-8') data = packer.pack(td) - re = Unpacker(BytesIO(data), encoding='utf-8', use_list=1).unpack() + re = Unpacker(BytesIO(data), encoding=str('utf-8'), use_list=1).unpack() assert re == td def testPackUTF32(): try: test_data = [ - six.u(""), - six.u("abcd"), - [six.u("defgh")], - six.u("РуÑÑкий текÑÑ‚"), + "", + "abcd", + ["defgh"], + "РуÑÑкий текÑÑ‚", ] for td in test_data: re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32') @@ -70,23 +68,23 @@ def testStrictUnicodeUnpack(): def testStrictUnicodePack(): with raises(UnicodeEncodeError): - packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict') + packb("abc\xeddef", encoding='ascii', unicode_errors='strict') def testIgnoreErrorsPack(): - re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1) - assert re == six.u("abcdef") + re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1) + assert re == "abcdef" def testNoEncoding(): with raises(TypeError): - packb(six.u("abc"), encoding=None) + packb("abc", encoding=None) def testDecodeBinary(): - re = unpackb(packb("abc"), encoding=None, use_list=1) + re = unpackb(packb(b"abc"), encoding=None, use_list=1) assert re == b"abc" def testPackFloat(): - assert packb(1.0, use_single_float=True) == b'\xca' + struct.pack('>f', 1.0) - assert packb(1.0, use_single_float=False) == b'\xcb' + struct.pack('>d', 1.0) + assert packb(1.0, use_single_float=True) == b'\xca' + struct.pack(str('>f'), 1.0) + assert packb(1.0, use_single_float=False) == b'\xcb' + struct.pack(str('>d'), 1.0) def testArraySize(sizes=[0, 5, 50, 1000]): bio = BytesIO() diff --git a/tox.ini b/tox.ini index 2c5c596..2413630 100644 --- a/tox.ini +++ b/tox.ini @@ -4,6 +4,5 @@ envlist = py26,py27,py32,py33,pypy [testenv] deps= pytest - six commands=py.test test From 7effb4aac64f1c44241e0a88ab19fa6c8405c3fa Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 17 Feb 2014 04:05:04 +0900 Subject: [PATCH 0758/1172] fix --- msgpack/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index b673222..0c5b50f 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -194,7 +194,7 @@ class Unpacker(object): if isinstance(next_bytes, array.array): next_bytes = next_bytes.tostring() elif isinstance(next_bytes, bytearray): - next_bytes = (bytes if PY3 else str)(next_bytes) + next_bytes = bytes(next_bytes) assert self._fb_feeding if self._fb_buf_n + len(next_bytes) > self._max_buffer_size: raise BufferFull From 518f886b111808d7d9f29b34c50baddb01610338 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 17 Feb 2014 04:06:58 +0900 Subject: [PATCH 0759/1172] fix --- test/test_buffer.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/test_buffer.py b/test/test_buffer.py index 7b85e93..6cb2295 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -8,7 +8,7 @@ import sys def test_unpack_buffer(): from array import array buf = array('b') - buf.fromstring(packb(('foo', 'bar'))) + buf.fromstring(packb((b'foo', b'bar'))) obj = unpackb(buf, use_list=1) assert [b'foo', b'bar'] == obj @@ -17,7 +17,5 @@ def test_unpack_bytearray(): buf = bytearray(packb(('foo', 'bar'))) obj = unpackb(buf, use_list=1) assert [b'foo', b'bar'] == obj - expected_type = bytes if sys.version_info[0] == 3 else str - assert all(type(s)==expected_type for s in obj) - - + expected_type = bytes + assert all(type(s) == expected_type for s in obj) From c567cf478b63baa3226e031bf988b6db2bf6af11 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 17 Feb 2014 09:54:55 +0900 Subject: [PATCH 0760/1172] Remove unused import. --- test/test_buffer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_buffer.py b/test/test_buffer.py index 6cb2295..5a71f90 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -2,7 +2,6 @@ # coding: utf-8 from msgpack import packb, unpackb -import sys def test_unpack_buffer(): From 1ca3c27a81eeb5b97fcd817cf4465923eeceea93 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 17 Feb 2014 10:03:36 +0900 Subject: [PATCH 0761/1172] 0.4.1 --- ChangeLog.rst | 18 ++++++++++++++++++ msgpack/_version.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index c5cff15..080f5c8 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,21 @@ +0.4.1 +===== +:release date: 2014-02-17 + +Inconpatible Changes +-------------------- + +Changes +------- + +* fallback.Unpacker.feed() supports bytearray. + +Bugs fixed +---------- + +* Unpacker doesn't increment refcount of hooks. Hooks may be GCed while unpacking. +* Unpacker may read unfilled internal buffer. + 0.4.0 ===== :release date: 2013-10-21 diff --git a/msgpack/_version.py b/msgpack/_version.py index e7a935c..c6ebcf8 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 4, 0) +version = (0, 4, 1) From 55eab8b4d64ac50f7a448e64bbc0cb092c0ed18d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 17 Feb 2014 10:06:39 +0900 Subject: [PATCH 0762/1172] Update README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index c51e518..f99ed48 100644 --- a/README.rst +++ b/README.rst @@ -3,8 +3,8 @@ MessagePack for Python ======================= :author: INADA Naoki -:version: 0.4.0 -:date: 2013-10-21 +:version: 0.4.1 +:date: 2014-02-17 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png :target: https://travis-ci.org/#!/msgpack/msgpack-python From e9de6b7f391a4b5e692af37505c959eaf02943e0 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 19 Mar 2014 10:42:05 +0900 Subject: [PATCH 0763/1172] travis: Add Python 3.4 to testing. --- .travis.yml | 4 ++++ tox.ini | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7cf7f58..1c1efc3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,10 @@ language: python python: - 2.7 +before_install: + - sudo apt-get update -qq -y + - sudo apt-get install python3.4 + install: - pip install tox cython - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx diff --git a/tox.ini b/tox.ini index 2413630..13d6b8d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,py32,py33,pypy +envlist = py26,py27,py32,py33,py34,pypy [testenv] deps= From d850e56dd0e2d392bb6ead388c0f0a59f68e1bd2 Mon Sep 17 00:00:00 2001 From: Alexey Popravka Date: Mon, 24 Mar 2014 15:31:06 +0200 Subject: [PATCH 0764/1172] Unpacker's ext_hook fixed + tests --- msgpack/_unpacker.pyx | 3 ++- test/test_unpack.py | 25 ++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 0df6ab3..16de40f 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -207,7 +207,7 @@ cdef class Unpacker(object): cdef object file_like_read cdef Py_ssize_t read_size # To maintain refcnt. - cdef object object_hook, object_pairs_hook, list_hook + cdef object object_hook, object_pairs_hook, list_hook, ext_hook cdef object encoding, unicode_errors cdef size_t max_buffer_size @@ -228,6 +228,7 @@ cdef class Unpacker(object): self.object_hook = object_hook self.object_pairs_hook = object_pairs_hook self.list_hook = list_hook + self.ext_hook = ext_hook self.file_like = file_like if file_like: diff --git a/test/test_unpack.py b/test/test_unpack.py index 544cebf..275f124 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -1,6 +1,6 @@ from io import BytesIO import sys -from msgpack import Unpacker, packb, OutOfData +from msgpack import Unpacker, packb, OutOfData, ExtType from pytest import raises, mark @@ -42,6 +42,29 @@ def test_unpacker_hook_refcnt(): assert sys.getrefcount(hook) == basecnt +def test_unpacker_ext_hook(): + + class MyUnpacker(Unpacker): + + def __init__(self): + super().__init__(ext_hook=self._hook, encoding='utf-8') + + def _hook(self, code, data): + if code == 1: + return int(data) + else: + return ExtType(code, data) + + unpacker = MyUnpacker() + unpacker.feed(packb({'a': 1}, encoding='utf-8')) + assert unpacker.unpack() == {'a': 1} + unpacker.feed(packb({'a': ExtType(1, b'123')}, encoding='utf-8')) + assert unpacker.unpack() == {'a': 123} + unpacker.feed(packb({'a': ExtType(2, b'321')}, encoding='utf-8')) + assert unpacker.unpack() == {'a': ExtType(2, b'321')} + + if __name__ == '__main__': test_unpack_array_header_from_file() test_unpacker_hook_refcnt() + test_unpacker_ext_hook() From ee38505db59c55d2d96516274030eed712028039 Mon Sep 17 00:00:00 2001 From: Alexey Popravka Date: Mon, 24 Mar 2014 15:42:16 +0200 Subject: [PATCH 0765/1172] fixed super() for python2 --- test/test_unpack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_unpack.py b/test/test_unpack.py index 275f124..8d0d949 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -47,7 +47,8 @@ def test_unpacker_ext_hook(): class MyUnpacker(Unpacker): def __init__(self): - super().__init__(ext_hook=self._hook, encoding='utf-8') + super(MyUnpacker, self).__init__(ext_hook=self._hook, + encoding='utf-8') def _hook(self, code, data): if code == 1: From 6c0c306f966c5cb1caeb2d6b0712ae3d0edd4d82 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 02:49:03 +0900 Subject: [PATCH 0766/1172] Add tests for limits. --- test/test_limits.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 test/test_limits.py diff --git a/test/test_limits.py b/test/test_limits.py new file mode 100644 index 0000000..970f722 --- /dev/null +++ b/test/test_limits.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# coding: utf-8 +import pytest + +from msgpack import packb, unpackb + + +def test_integer(): + x = -(2 ** 63) + assert unpackb(packb(x)) == x + with pytest.raises(OverflowError): + packb(x-1) + + x = 2 ** 64 - 1 + assert unpackb(packb(x)) == x + with pytest.raises(OverflowError): + packb(x+1) + +@pytest.mark.skipif(True, "Requires very large memory.") +def test_binary(): + x = b'x' * (2**32 - 1) + assert unpackb(packb(x)) == x + x += b'y' + with pytest.raises(ValueError): + packb(x) + + +@pytest.mark.skipif(True, "Requires very large memory.") +def test_string(): + x = u'x' * (2**32 - 1) + assert unpackb(packb(x)) == x + x += u'y' + with pytest.raises(ValueError): + packb(x) + + +@pytest.mark.skipif(True, "Requires very large memory.") +def test_array(): + x = [0] * (2**32 - 1) + assert unpackb(packb(x)) == x + x.append(0) + with pytest.raises(ValueError): + packb(x) From e7f87d9d41532b0956ebcd9f7be52df979842466 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 03:00:47 +0900 Subject: [PATCH 0767/1172] More limit check. --- test/test_limits.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/test/test_limits.py b/test/test_limits.py index 970f722..2879c0a 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -2,7 +2,7 @@ # coding: utf-8 import pytest -from msgpack import packb, unpackb +from msgpack import packb, unpackb, Packer def test_integer(): @@ -16,11 +16,27 @@ def test_integer(): with pytest.raises(OverflowError): packb(x+1) + +def test_array_header(): + packer = Packer() + packer.pack_array_header(2**32-1) + with pytest.raises(ValueError): + packer.pack_array_header(2**32) + + +def test_map_header(): + packer = Packer() + packer.pack_map_header(2**32-1) + with pytest.raises(ValueError): + packer.pack_array_header(2**32) + + @pytest.mark.skipif(True, "Requires very large memory.") def test_binary(): x = b'x' * (2**32 - 1) assert unpackb(packb(x)) == x - x += b'y' + del x + x = b'x' * (2**32) with pytest.raises(ValueError): packb(x) From c60ab28ee72e1b179e21f5fbc6efd0f96d6e6467 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 03:03:18 +0900 Subject: [PATCH 0768/1172] Add check for format limits. --- msgpack/_packer.pyx | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index f261f08..204beaa 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -135,6 +135,8 @@ cdef class Packer(object): else: ret = msgpack_pack_false(&self.pk) elif PyLong_Check(o): + # PyInt_Check(long) is True for Python 3. + # Sow we should test long before int. if o > 0: ullval = o ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) @@ -152,8 +154,10 @@ cdef class Packer(object): dval = o ret = msgpack_pack_double(&self.pk, dval) elif PyBytes_Check(o): - rawval = o L = len(o) + if L > (2**32)-1: + raise ValueError("bytes is too large") + rawval = o ret = msgpack_pack_bin(&self.pk, L) if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, L) @@ -161,13 +165,19 @@ cdef class Packer(object): if not self.encoding: raise TypeError("Can't encode unicode string: no encoding is specified") o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) + L = len(o) + if L > (2**32)-1: + raise ValueError("dict is too large") rawval = o ret = msgpack_pack_raw(&self.pk, len(o)) if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyDict_CheckExact(o): d = o - ret = msgpack_pack_map(&self.pk, len(d)) + L = len(d) + if L > (2**32)-1: + raise ValueError("dict is too large") + ret = msgpack_pack_map(&self.pk, L) if ret == 0: for k, v in d.iteritems(): ret = self._pack(k, nest_limit-1) @@ -175,7 +185,10 @@ cdef class Packer(object): ret = self._pack(v, nest_limit-1) if ret != 0: break elif PyDict_Check(o): - ret = msgpack_pack_map(&self.pk, len(o)) + L = len(o) + if L > (2**32)-1: + raise ValueError("dict is too large") + ret = msgpack_pack_map(&self.pk, L) if ret == 0: for k, v in o.items(): ret = self._pack(k, nest_limit-1) @@ -187,10 +200,15 @@ cdef class Packer(object): longval = o.code rawval = o.data L = len(o.data) + if L > (2**32)-1: + raise ValueError("EXT data is too large") ret = msgpack_pack_ext(&self.pk, longval, L) ret = msgpack_pack_raw_body(&self.pk, rawval, L) elif PyTuple_Check(o) or PyList_Check(o): - ret = msgpack_pack_array(&self.pk, len(o)) + L = len(o) + if L > (2**32)-1: + raise ValueError("list is too large") + ret = msgpack_pack_array(&self.pk, L) if ret == 0: for v in o: ret = self._pack(v, nest_limit-1) @@ -220,6 +238,8 @@ cdef class Packer(object): msgpack_pack_raw_body(&self.pk, data, len(data)) def pack_array_header(self, size_t size): + if size >= (2**32-1): + raise ValueError cdef int ret = msgpack_pack_array(&self.pk, size) if ret == -1: raise MemoryError @@ -231,6 +251,8 @@ cdef class Packer(object): return buf def pack_map_header(self, size_t size): + if size >= (2**32-1): + raise ValueError cdef int ret = msgpack_pack_map(&self.pk, size) if ret == -1: raise MemoryError From e99331d1ab793105da2e0a4139059169e5c835b3 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 11:01:44 +0900 Subject: [PATCH 0769/1172] Fix skipif marking. --- test/test_limits.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_limits.py b/test/test_limits.py index 2879c0a..5d285e7 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -31,7 +31,7 @@ def test_map_header(): packer.pack_array_header(2**32) -@pytest.mark.skipif(True, "Requires very large memory.") +@pytest.mark.skipif(True, reason="Requires very large memory.") def test_binary(): x = b'x' * (2**32 - 1) assert unpackb(packb(x)) == x @@ -41,7 +41,7 @@ def test_binary(): packb(x) -@pytest.mark.skipif(True, "Requires very large memory.") +@pytest.mark.skipif(True, reason="Requires very large memory.") def test_string(): x = u'x' * (2**32 - 1) assert unpackb(packb(x)) == x @@ -50,7 +50,7 @@ def test_string(): packb(x) -@pytest.mark.skipif(True, "Requires very large memory.") +@pytest.mark.skipif(True, reason="Requires very large memory.") def test_array(): x = [0] * (2**32 - 1) assert unpackb(packb(x)) == x From ef5d93d4eaaaf107094776c215f07efd0faae3e9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 11:05:53 +0900 Subject: [PATCH 0770/1172] Fix size limit on pack_array_header and pack_map_header. --- msgpack/_packer.pyx | 4 ++-- msgpack/fallback.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 204beaa..82e4a63 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -238,7 +238,7 @@ cdef class Packer(object): msgpack_pack_raw_body(&self.pk, data, len(data)) def pack_array_header(self, size_t size): - if size >= (2**32-1): + if size > (2**32-1): raise ValueError cdef int ret = msgpack_pack_array(&self.pk, size) if ret == -1: @@ -251,7 +251,7 @@ cdef class Packer(object): return buf def pack_map_header(self, size_t size): - if size >= (2**32-1): + if size > (2**32-1): raise ValueError cdef int ret = msgpack_pack_map(&self.pk, size) if ret == -1: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 55de72c..49323e6 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -633,6 +633,8 @@ class Packer(object): return ret def pack_array_header(self, n): + if n >= 2**32: + raise ValueError self._fb_pack_array_header(n) ret = self._buffer.getvalue() if self._autoreset: @@ -642,6 +644,8 @@ class Packer(object): return ret def pack_map_header(self, n): + if n >= 2**32: + raise ValueError self._fb_pack_map_header(n) ret = self._buffer.getvalue() if self._autoreset: From 7d0e145e91700ee58c5accaeb5dff6a9ec39a18d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 12:50:28 +0900 Subject: [PATCH 0771/1172] Allow ValueError for packing integer overs format limit. --- test/test_limits.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_limits.py b/test/test_limits.py index 5d285e7..413d651 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -8,12 +8,12 @@ from msgpack import packb, unpackb, Packer def test_integer(): x = -(2 ** 63) assert unpackb(packb(x)) == x - with pytest.raises(OverflowError): + with pytest.raises((OverflowError, ValueError)): packb(x-1) x = 2 ** 64 - 1 assert unpackb(packb(x)) == x - with pytest.raises(OverflowError): + with pytest.raises((OverflowError, ValueError)): packb(x+1) From 5fb9d8a7fd09893afdd1e5844288313bb347856b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 13:32:28 +0900 Subject: [PATCH 0772/1172] Fix Python 3.2 --- test/test_limits.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test_limits.py b/test/test_limits.py index 413d651..7bc8f4c 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # coding: utf-8 +from __future__ import absolute_import, division, print_function, unicode_literals import pytest from msgpack import packb, unpackb, Packer @@ -43,9 +44,9 @@ def test_binary(): @pytest.mark.skipif(True, reason="Requires very large memory.") def test_string(): - x = u'x' * (2**32 - 1) + x = 'x' * (2**32 - 1) assert unpackb(packb(x)) == x - x += u'y' + x += 'y' with pytest.raises(ValueError): packb(x) From a72e75d7c81486bc2775b957b22d3b5e13742589 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 15:12:28 +0900 Subject: [PATCH 0773/1172] Fix PyPy fail. --- test/test_limits.py | 54 +++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/test/test_limits.py b/test/test_limits.py index 7bc8f4c..da8cd2b 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -32,29 +32,31 @@ def test_map_header(): packer.pack_array_header(2**32) -@pytest.mark.skipif(True, reason="Requires very large memory.") -def test_binary(): - x = b'x' * (2**32 - 1) - assert unpackb(packb(x)) == x - del x - x = b'x' * (2**32) - with pytest.raises(ValueError): - packb(x) - - -@pytest.mark.skipif(True, reason="Requires very large memory.") -def test_string(): - x = 'x' * (2**32 - 1) - assert unpackb(packb(x)) == x - x += 'y' - with pytest.raises(ValueError): - packb(x) - - -@pytest.mark.skipif(True, reason="Requires very large memory.") -def test_array(): - x = [0] * (2**32 - 1) - assert unpackb(packb(x)) == x - x.append(0) - with pytest.raises(ValueError): - packb(x) +# PyPy fails following tests because of constant folding? +# https://bugs.pypy.org/issue1721 +#@pytest.mark.skipif(True, reason="Requires very large memory.") +#def test_binary(): +# x = b'x' * (2**32 - 1) +# assert unpackb(packb(x)) == x +# del x +# x = b'x' * (2**32) +# with pytest.raises(ValueError): +# packb(x) +# +# +#@pytest.mark.skipif(True, reason="Requires very large memory.") +#def test_string(): +# x = 'x' * (2**32 - 1) +# assert unpackb(packb(x)) == x +# x += 'y' +# with pytest.raises(ValueError): +# packb(x) +# +# +#@pytest.mark.skipif(True, reason="Requires very large memory.") +#def test_array(): +# x = [0] * (2**32 - 1) +# assert unpackb(packb(x)) == x +# x.append(0) +# with pytest.raises(ValueError): +# packb(x) From 803684b90debbd9b4f8a2631611c613e138cd7df Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 15:52:03 +0900 Subject: [PATCH 0774/1172] 0.4.2 --- ChangeLog.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 080f5c8..610b026 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,19 @@ +0.4.2 +===== +:release date: 2014-03-26 + +Inconpatible Changes +-------------------- + +Changes +------- + +Bugs fixed +---------- + +* Unpacker doesn't increment refcount of ExtType hook. +* Packer raises no exception for inputs doesn't fit to msgpack format. + 0.4.1 ===== :release date: 2014-02-17 From 61bac2f586e82313a0e618093bfed2435cd18983 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 26 Mar 2014 17:20:51 +0900 Subject: [PATCH 0775/1172] 0.4.2 --- msgpack/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index c6ebcf8..dddfe49 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 4, 1) +version = (0, 4, 2) From 3b933f0966b1e53ea50418970950de294ebbea76 Mon Sep 17 00:00:00 2001 From: faerot Date: Thu, 22 May 2014 11:32:54 +0300 Subject: [PATCH 0776/1172] added distinguish_tuple argument to Packer This will make precise python types serialization possible. --- msgpack/_packer.pyx | 12 ++++++++++-- msgpack/fallback.py | 11 +++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 82e4a63..86e460f 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -56,6 +56,10 @@ cdef class Packer(object): Convert unicode to bytes with this encoding. (default: 'utf-8') :param str unicode_errors: Error handler for encoding unicode. (default: 'strict') + :param bool distinguish_tuple: + If set to true, tuples will not be serialized as lists + and will be treated as unsupported type. This is useful when trying + to implement accurate serialization for python types. :param bool use_single_float: Use single precision float type for float. (default: False) :param bool autoreset: @@ -71,6 +75,7 @@ cdef class Packer(object): cdef object _berrors cdef char *encoding cdef char *unicode_errors + cdef bool distinguish_tuple cdef bool use_float cdef bint autoreset @@ -83,10 +88,12 @@ cdef class Packer(object): self.pk.length = 0 def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', - use_single_float=False, bint autoreset=1, bint use_bin_type=0): + distinguish_tuple=False, use_single_float=False, bint autoreset=1, + bint use_bin_type=0): """ """ self.use_float = use_single_float + self.distinguish_tuple = distinguish_tuple self.autoreset = autoreset self.pk.use_bin_type = use_bin_type if default is not None: @@ -122,6 +129,7 @@ cdef class Packer(object): cdef dict d cdef size_t L cdef int default_used = 0 + cdef bool distinguish_tuple = self.distinguish_tuple if nest_limit < 0: raise PackValueError("recursion limit exceeded.") @@ -204,7 +212,7 @@ cdef class Packer(object): raise ValueError("EXT data is too large") ret = msgpack_pack_ext(&self.pk, longval, L) ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyTuple_Check(o) or PyList_Check(o): + elif (PyTuple_Check(o) and not distinguish_tuple) or PyList_Check(o): L = len(o) if L > (2**32)-1: raise ValueError("list is too large") diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 49323e6..1d668c2 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -485,6 +485,10 @@ class Packer(object): Convert unicode to bytes with this encoding. (default: 'utf-8') :param str unicode_errors: Error handler for encoding unicode. (default: 'strict') + :param bool distinguish_tuple: + If set to true, tuples will not be serialized as lists + and will be treated as unsupported type. This is useful when trying + to implement accurate serialization for python types. :param bool use_single_float: Use single precision float type for float. (default: False) :param bool autoreset: @@ -495,7 +499,9 @@ class Packer(object): It also enable str8 type for unicode. """ def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', - use_single_float=False, autoreset=True, use_bin_type=False): + distinguish_tuple=False, use_single_float=False, autoreset=True, + use_bin_type=False): + self._distinguish_tuple = distinguish_tuple self._use_float = use_single_float self._autoreset = autoreset self._use_bin_type = use_bin_type @@ -509,6 +515,7 @@ class Packer(object): def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance): default_used = False + list_type = list if self._distinguish_tuple else (list, tuple) while True: if nest_limit < 0: raise PackValueError("recursion limit exceeded") @@ -599,7 +606,7 @@ class Packer(object): self._buffer.write(struct.pack("b", code)) self._buffer.write(data) return - if isinstance(obj, (list, tuple)): + if isinstance(obj, list_type): n = len(obj) self._fb_pack_array_header(n) for i in xrange(n): From b877ce2afadd4a4c96d7c0542f7b29836785de71 Mon Sep 17 00:00:00 2001 From: faerot Date: Thu, 22 May 2014 16:45:26 +0300 Subject: [PATCH 0777/1172] precise_mode instead of distinguish_tuple When precise_mode flag is set, serialization will be as precise as possible - type checks will be exact (type(..) is ... instead of isinstance(..., ...) and tuple will be treated as undefined type. This mode is to make accurate object serialization possible. --- msgpack/_packer.pyx | 37 ++++++++++++++++++---------------- msgpack/fallback.py | 48 +++++++++++++++++++++++++++++---------------- 2 files changed, 51 insertions(+), 34 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 86e460f..ec34cd8 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -56,10 +56,13 @@ cdef class Packer(object): Convert unicode to bytes with this encoding. (default: 'utf-8') :param str unicode_errors: Error handler for encoding unicode. (default: 'strict') - :param bool distinguish_tuple: - If set to true, tuples will not be serialized as lists - and will be treated as unsupported type. This is useful when trying - to implement accurate serialization for python types. + :param bool precise_mode: + If set to true, types will be checked to be exact. Derived classes + from serializeable types will not be serialized and will be + treated as unsupported type and forwarded to default. + Additionally tuples will not be serialized as lists. + This is useful when trying to implement accurate serialization + for python types. :param bool use_single_float: Use single precision float type for float. (default: False) :param bool autoreset: @@ -75,7 +78,7 @@ cdef class Packer(object): cdef object _berrors cdef char *encoding cdef char *unicode_errors - cdef bool distinguish_tuple + cdef bint precise_mode cdef bool use_float cdef bint autoreset @@ -88,12 +91,12 @@ cdef class Packer(object): self.pk.length = 0 def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', - distinguish_tuple=False, use_single_float=False, bint autoreset=1, - bint use_bin_type=0): + use_single_float=False, bint autoreset=1, bint use_bin_type=0, + bint precise_mode=0): """ """ self.use_float = use_single_float - self.distinguish_tuple = distinguish_tuple + self.precise_mode = precise_mode self.autoreset = autoreset self.pk.use_bin_type = use_bin_type if default is not None: @@ -129,7 +132,7 @@ cdef class Packer(object): cdef dict d cdef size_t L cdef int default_used = 0 - cdef bool distinguish_tuple = self.distinguish_tuple + cdef bint precise = self.precise_mode if nest_limit < 0: raise PackValueError("recursion limit exceeded.") @@ -137,12 +140,12 @@ cdef class Packer(object): while True: if o is None: ret = msgpack_pack_nil(&self.pk) - elif isinstance(o, bool): + elif PyBool_Check(o) if precise else isinstance(o, bool): if o: ret = msgpack_pack_true(&self.pk) else: ret = msgpack_pack_false(&self.pk) - elif PyLong_Check(o): + elif PyLong_CheckExact(o) if precise else PyLong_Check(o): # PyInt_Check(long) is True for Python 3. # Sow we should test long before int. if o > 0: @@ -151,17 +154,17 @@ cdef class Packer(object): else: llval = o ret = msgpack_pack_long_long(&self.pk, llval) - elif PyInt_Check(o): + elif PyInt_CheckExact(o) if precise else PyInt_Check(o): longval = o ret = msgpack_pack_long(&self.pk, longval) - elif PyFloat_Check(o): + elif PyFloat_CheckExact(o) if precise else PyFloat_Check(o): if self.use_float: fval = o ret = msgpack_pack_float(&self.pk, fval) else: dval = o ret = msgpack_pack_double(&self.pk, dval) - elif PyBytes_Check(o): + elif PyBytes_CheckExact(o) if precise else PyBytes_Check(o): L = len(o) if L > (2**32)-1: raise ValueError("bytes is too large") @@ -169,7 +172,7 @@ cdef class Packer(object): ret = msgpack_pack_bin(&self.pk, L) if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyUnicode_Check(o): + elif PyUnicode_CheckExact(o) if precise else PyUnicode_Check(o): if not self.encoding: raise TypeError("Can't encode unicode string: no encoding is specified") o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) @@ -192,7 +195,7 @@ cdef class Packer(object): if ret != 0: break ret = self._pack(v, nest_limit-1) if ret != 0: break - elif PyDict_Check(o): + elif not precise and PyDict_Check(o): L = len(o) if L > (2**32)-1: raise ValueError("dict is too large") @@ -212,7 +215,7 @@ cdef class Packer(object): raise ValueError("EXT data is too large") ret = msgpack_pack_ext(&self.pk, longval, L) ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif (PyTuple_Check(o) and not distinguish_tuple) or PyList_Check(o): + elif PyList_CheckExact(o) if precise else (PyTuple_Check(o) or PyList_Check(o)): L = len(o) if L > (2**32)-1: raise ValueError("list is too large") diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 1d668c2..77922f7 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -485,10 +485,13 @@ class Packer(object): Convert unicode to bytes with this encoding. (default: 'utf-8') :param str unicode_errors: Error handler for encoding unicode. (default: 'strict') - :param bool distinguish_tuple: - If set to true, tuples will not be serialized as lists - and will be treated as unsupported type. This is useful when trying - to implement accurate serialization for python types. + :param bool precise_mode: + If set to true, types will be checked to be exact. Derived classes + from serializeable types will not be serialized and will be + treated as unsupported type and forwarded to default. + Additionally tuples will not be serialized as lists. + This is useful when trying to implement accurate serialization + for python types. :param bool use_single_float: Use single precision float type for float. (default: False) :param bool autoreset: @@ -499,9 +502,9 @@ class Packer(object): It also enable str8 type for unicode. """ def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', - distinguish_tuple=False, use_single_float=False, autoreset=True, + precise_mode=False, use_single_float=False, autoreset=True, use_bin_type=False): - self._distinguish_tuple = distinguish_tuple + self._precise_mode = precise_mode self._use_float = use_single_float self._autoreset = autoreset self._use_bin_type = use_bin_type @@ -513,19 +516,30 @@ class Packer(object): raise TypeError("default must be callable") self._default = default - def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance): + def _check_precise(obj, t, type=type, tuple=tuple): + if type(t) is tuple: + return type(obj) in t + else: + return type(obj) is t + + def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, + check=isinstance, check_precise=_check_precise): default_used = False - list_type = list if self._distinguish_tuple else (list, tuple) + if self._precise_mode: + check = check_precise + list_types = list + else: + list_types = (list, tuple) while True: if nest_limit < 0: raise PackValueError("recursion limit exceeded") if obj is None: return self._buffer.write(b"\xc0") - if isinstance(obj, bool): + if check(obj, bool): if obj: return self._buffer.write(b"\xc3") return self._buffer.write(b"\xc2") - if isinstance(obj, int_types): + if check(obj, int_types): if 0 <= obj < 0x80: return self._buffer.write(struct.pack("B", obj)) if -0x20 <= obj < 0: @@ -547,7 +561,7 @@ class Packer(object): if -0x8000000000000000 <= obj < -0x80000000: return self._buffer.write(struct.pack(">Bq", 0xd3, obj)) raise PackValueError("Integer value out of range") - if self._use_bin_type and isinstance(obj, bytes): + if self._use_bin_type and check(obj, bytes): n = len(obj) if n <= 0xff: self._buffer.write(struct.pack('>BB', 0xc4, n)) @@ -558,8 +572,8 @@ class Packer(object): else: raise PackValueError("Bytes is too large") return self._buffer.write(obj) - if isinstance(obj, (Unicode, bytes)): - if isinstance(obj, Unicode): + if check(obj, (Unicode, bytes)): + if check(obj, Unicode): if self._encoding is None: raise TypeError( "Can't encode unicode string: " @@ -577,11 +591,11 @@ class Packer(object): else: raise PackValueError("String is too large") return self._buffer.write(obj) - if isinstance(obj, float): + if check(obj, float): if self._use_float: return self._buffer.write(struct.pack(">Bf", 0xca, obj)) return self._buffer.write(struct.pack(">Bd", 0xcb, obj)) - if isinstance(obj, ExtType): + if check(obj, ExtType): code = obj.code data = obj.data assert isinstance(code, int) @@ -606,13 +620,13 @@ class Packer(object): self._buffer.write(struct.pack("b", code)) self._buffer.write(data) return - if isinstance(obj, list_type): + if check(obj, list_types): n = len(obj) self._fb_pack_array_header(n) for i in xrange(n): self._pack(obj[i], nest_limit - 1) return - if isinstance(obj, dict): + if check(obj, dict): return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj), nest_limit - 1) if not default_used and self._default is not None: From 7f623c09060fd2561efaa6e5b4ef40da9747ec9e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 26 May 2014 01:17:53 +0900 Subject: [PATCH 0778/1172] Fix unpacking uint32 on 32bit or LLP64. --- msgpack/unpack.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 27e3b62..71142c6 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -62,7 +62,7 @@ static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unp } else #endif { - p = PyInt_FromLong((long)d); + p = PyInt_FromUnsignedLong((long)d); } if (!p) return -1; From 8f1c0504f15440ba029ca2a069da17db7cb31f65 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 26 May 2014 01:28:30 +0900 Subject: [PATCH 0779/1172] Travis preinstall Python 3.4 --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1c1efc3..7cf7f58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,6 @@ language: python python: - 2.7 -before_install: - - sudo apt-get update -qq -y - - sudo apt-get install python3.4 - install: - pip install tox cython - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx From 4d4a0cc4425b38b1a928ef80ee8963a6b259c316 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 26 May 2014 15:21:14 +0900 Subject: [PATCH 0780/1172] Add changelog --- ChangeLog.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 610b026..a7b9800 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,18 @@ +0.4.3 +===== +:release date: TBD + +Inconpatible Changes +-------------------- + +Changes +------- + +Bugs fixed +---------- + +* Unpacker may unpack wrong uint32 value on 32bit or LLP64 environment. (#101) + 0.4.2 ===== :release date: 2014-03-26 From 67391fd60e2285067a6cbd7392ef0a603875cc59 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Sat, 14 Jun 2014 18:25:34 +0200 Subject: [PATCH 0781/1172] fallback: add missing update of _fb_buf_n --- msgpack/fallback.py | 1 + 1 file changed, 1 insertion(+) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 49323e6..d326885 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -252,6 +252,7 @@ class Unpacker(object): if not tmp: break buffs.append(tmp) + self._fb_buf_n += len(tmp) continue sliced = n - len(ret) ret += buffs[self._fb_buf_i][self._fb_buf_o:self._fb_buf_o + sliced] From 56cf3841593733dafe0f84a0e8cbc16c3dc7250d Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Sat, 14 Jun 2014 18:25:57 +0200 Subject: [PATCH 0782/1172] fallback: set default read_size to 4096 --- msgpack/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index d326885..294a97e 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -169,7 +169,7 @@ class Unpacker(object): self._max_buffer_size = max_buffer_size or 2**31-1 if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") - self._read_size = read_size or min(self._max_buffer_size, 2048) + self._read_size = read_size or min(self._max_buffer_size, 4096) self._encoding = encoding self._unicode_errors = unicode_errors self._use_list = use_list From ba8cf1c402fe68f283480c9abbf7707d004a8583 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Sat, 14 Jun 2014 18:26:30 +0200 Subject: [PATCH 0783/1172] fallback: _fb_consume: improve performance with pypy --- msgpack/fallback.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 294a97e..0e0557f 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -202,12 +202,17 @@ class Unpacker(object): self._fb_buffers.append(next_bytes) def _fb_consume(self): - self._fb_buffers = self._fb_buffers[self._fb_buf_i:] + if self._fb_buf_i: + for i in xrange(self._fb_buf_i): + self._fb_buf_n -= len(self._fb_buffers[i]) + self._fb_buffers = self._fb_buffers[self._fb_buf_i:] + self._fb_buf_i = 0 if self._fb_buffers: self._fb_buffers[0] = self._fb_buffers[0][self._fb_buf_o:] + self._fb_buf_n -= self._fb_buf_o + else: + self._fb_buf_n = 0 self._fb_buf_o = 0 - self._fb_buf_i = 0 - self._fb_buf_n = sum(map(len, self._fb_buffers)) def _fb_got_extradata(self): if self._fb_buf_i != len(self._fb_buffers): From 7eb371f8278941fb2323e0c2333ed89c88ab822b Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Sat, 14 Jun 2014 18:30:38 +0200 Subject: [PATCH 0784/1172] fallback: do not reset the buffer completely in between of iterations --- msgpack/fallback.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 0e0557f..d838e55 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -201,7 +201,16 @@ class Unpacker(object): self._fb_buf_n += len(next_bytes) self._fb_buffers.append(next_bytes) + def _fb_sloppy_consume(self): + """ Gets rid of some of the used parts of the buffer. """ + if self._fb_buf_i: + for i in xrange(self._fb_buf_i): + self._fb_buf_n -= len(self._fb_buffers[i]) + self._fb_buffers = self._fb_buffers[self._fb_buf_i:] + self._fb_buf_i = 0 + def _fb_consume(self): + """ Gets rid of the used parts of the buffer. """ if self._fb_buf_i: for i in xrange(self._fb_buf_i): self._fb_buf_n -= len(self._fb_buffers[i]) @@ -446,9 +455,10 @@ class Unpacker(object): def next(self): try: ret = self._fb_unpack(EX_CONSTRUCT, None) - self._fb_consume() + self._fb_sloppy_consume() return ret except OutOfData: + self._fb_consume() raise StopIteration __next__ = next From 952eb9fc5331fb7134cc70ce6084b30eff4ccda2 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Sat, 14 Jun 2014 18:34:17 +0200 Subject: [PATCH 0785/1172] fallback: add some comments to _fb_read --- msgpack/fallback.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index d838e55..fbd87ba 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -252,11 +252,15 @@ class Unpacker(object): def _fb_read(self, n, write_bytes=None): buffs = self._fb_buffers + # We have a redundant codepath for the most common case, such that + # pypy optimizes it properly. This is the case that the read fits + # in the current buffer. if (write_bytes is None and self._fb_buf_i < len(buffs) and self._fb_buf_o + n < len(buffs[self._fb_buf_i])): self._fb_buf_o += n return buffs[self._fb_buf_i][self._fb_buf_o - n:self._fb_buf_o] + # The remaining cases. ret = b'' while len(ret) != n: if self._fb_buf_i == len(buffs): From b334d441c3ed4e8bcd01bb1c2440dc975c377e56 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Sat, 14 Jun 2014 18:42:02 +0200 Subject: [PATCH 0786/1172] fallback: _fb_read: do a big read, when we need a big read --- msgpack/fallback.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index fbd87ba..c422fb5 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -263,16 +263,19 @@ class Unpacker(object): # The remaining cases. ret = b'' while len(ret) != n: + sliced = n - len(ret) if self._fb_buf_i == len(buffs): if self._fb_feeding: break - tmp = self.file_like.read(self._read_size) + to_read = sliced + if self._read_size > to_read: + to_read = self._read_size + tmp = self.file_like.read(to_read) if not tmp: break buffs.append(tmp) self._fb_buf_n += len(tmp) continue - sliced = n - len(ret) ret += buffs[self._fb_buf_i][self._fb_buf_o:self._fb_buf_o + sliced] self._fb_buf_o += sliced if self._fb_buf_o >= len(buffs[self._fb_buf_i]): From 0532ea87fb9b7f8d3b8544ff116fd589ad2a93b2 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Sun, 15 Jun 2014 22:45:30 +0200 Subject: [PATCH 0787/1172] fallback: fix BufferFull with sloppy consume --- msgpack/fallback.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index c422fb5..8ab3ad2 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -166,6 +166,11 @@ class Unpacker(object): self._fb_buf_o = 0 self._fb_buf_i = 0 self._fb_buf_n = 0 + # When Unpacker is used as an iterable, between the calls to next(), + # the buffer is not "consumed" completely, for efficiency sake. + # Instead, it is done sloppily. To make sure we raise BufferFull at + # the correct moments, we have to keep track of how sloppy we were. + self._fb_sloppiness = 0 self._max_buffer_size = max_buffer_size or 2**31-1 if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") @@ -196,7 +201,8 @@ class Unpacker(object): elif isinstance(next_bytes, bytearray): next_bytes = bytes(next_bytes) assert self._fb_feeding - if self._fb_buf_n + len(next_bytes) > self._max_buffer_size: + if (self._fb_buf_n + len(next_bytes) - self._fb_sloppiness + > self._max_buffer_size): raise BufferFull self._fb_buf_n += len(next_bytes) self._fb_buffers.append(next_bytes) @@ -208,6 +214,10 @@ class Unpacker(object): self._fb_buf_n -= len(self._fb_buffers[i]) self._fb_buffers = self._fb_buffers[self._fb_buf_i:] self._fb_buf_i = 0 + if self._fb_buffers: + self._fb_sloppiness = self._fb_buf_o + else: + self._fb_sloppiness = 0 def _fb_consume(self): """ Gets rid of the used parts of the buffer. """ @@ -222,6 +232,7 @@ class Unpacker(object): else: self._fb_buf_n = 0 self._fb_buf_o = 0 + self._fb_sloppiness = 0 def _fb_got_extradata(self): if self._fb_buf_i != len(self._fb_buffers): From d6c773dc4d19e4e8bfe2931d8daf251a8b302f25 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 31 Aug 2014 02:29:05 +0900 Subject: [PATCH 0788/1172] Fix build and tests. --- msgpack/unpack.h | 10 +--------- test/test_limits.py | 4 ++-- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 71142c6..24045d5 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -55,15 +55,7 @@ static inline int unpack_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpac static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { - PyObject *p; -#if UINT32_MAX > LONG_MAX - if (d > LONG_MAX) { - p = PyLong_FromUnsignedLong((unsigned long)d); - } else -#endif - { - p = PyInt_FromUnsignedLong((long)d); - } + PyObject *p = PyInt_FromSize_t((size_t)d); if (!p) return -1; *o = p; diff --git a/test/test_limits.py b/test/test_limits.py index da8cd2b..1cfa2d6 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -21,14 +21,14 @@ def test_integer(): def test_array_header(): packer = Packer() packer.pack_array_header(2**32-1) - with pytest.raises(ValueError): + with pytest.raises((OverflowError, ValueError)): packer.pack_array_header(2**32) def test_map_header(): packer = Packer() packer.pack_map_header(2**32-1) - with pytest.raises(ValueError): + with pytest.raises((OverflowError, ValueError)): packer.pack_array_header(2**32) From 6948dd51209f0f6be33d94fc3d44d20345ff9951 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 31 Aug 2014 03:03:48 +0900 Subject: [PATCH 0789/1172] Fix benchmark --- benchmark/benchmark.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 1439289..80819c6 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -15,10 +15,12 @@ def profile(name, func): def simple(name, data): if has_ext: - profile("packing %s (ext)" % name, lambda: _packer.packb(data)) - profile('packing %s (fallback)' % name, lambda: fallback.packb(data)) + packer = _packer.Packer() + profile("packing %s (ext)" % name, lambda: packer.pack(data)) + packer = fallback.Packer() + profile('packing %s (fallback)' % name, lambda: packer.pack(data)) - data = fallback.packb(data) + data = packer.pack(data) if has_ext: profile('unpacking %s (ext)' % name, lambda: _unpacker.unpackb(data)) profile('unpacking %s (fallback)' % name, lambda: fallback.unpackb(data)) From d5e9ac93162e3ea846be8862b54267fe9616f72b Mon Sep 17 00:00:00 2001 From: Xiaojie Lin Date: Fri, 14 Nov 2014 14:47:03 +1100 Subject: [PATCH 0790/1172] add support for pypy3 --- msgpack/fallback.py | 5 ++++- tox.ini | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 8ab3ad2..71fa7be 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -23,7 +23,10 @@ if hasattr(sys, 'pypy_version_info'): # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own # StringBuilder is fastest. from __pypy__ import newlist_hint - from __pypy__.builders import StringBuilder + try: + from __pypy__.builders import BytesBuilder as StringBuilder + except ImportError: + from __pypy__.builders import StringBuilder USING_STRINGBUILDER = True class StringIO(object): def __init__(self, s=b''): diff --git a/tox.ini b/tox.ini index 13d6b8d..892684d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,py32,py33,py34,pypy +envlist = py26,py27,py32,py33,py34,pypy,pypy3 [testenv] deps= From 0be3e874c63cfbdd647f6800e44a049657aef12b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 18 Dec 2014 13:17:49 +0900 Subject: [PATCH 0791/1172] Use new container based Travis-CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 7cf7f58..29f1b7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +sudo: false language: python python: - 2.7 From 593887025ea73acdd94fed93480011fb26e0aca8 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 2 Jan 2015 12:12:09 +0900 Subject: [PATCH 0792/1172] Fix README reST --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index f99ed48..705ad75 100644 --- a/README.rst +++ b/README.rst @@ -110,7 +110,7 @@ msgpack provides ``dumps`` and ``loads`` as alias for compatibility with >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False) (1, 2, 3) -You should always pass the ``use_list`` keyword argument. See performance issues relating to use_list_ below. +You should always pass the ``use_list`` keyword argument. See performance issues relating to `use_list option`_ below. Read the docstring for other options. @@ -237,8 +237,8 @@ CPython's GC starts when growing allocated object. This means unpacking may cause useless GC. You can use ``gc.disable()`` when unpacking large message. -`use_list` option -^^^^^^^^^^^^^^^^^^ +use_list option +^^^^^^^^^^^^^^^^ List is the default sequence type of Python. But tuple is lighter than list. You can use ``use_list=False`` while unpacking when performance is important. From 9624a2aca31ae94a92e2fc7225a12bd4875e3591 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 7 Jan 2015 12:10:42 +0900 Subject: [PATCH 0793/1172] Fix build failuer for Python 2.7 on Windows. Remove int8_t usage. --- msgpack/_packer.pyx | 3 +-- msgpack/pack.h | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 82e4a63..fcd20a7 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -5,7 +5,6 @@ from cpython cimport * from libc.stdlib cimport * from libc.string cimport * from libc.limits cimport * -from libc.stdint cimport int8_t from msgpack.exceptions import PackValueError from msgpack import ExtType @@ -32,7 +31,7 @@ cdef extern from "pack.h": int msgpack_pack_raw(msgpack_packer* pk, size_t l) int msgpack_pack_bin(msgpack_packer* pk, size_t l) int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) - int msgpack_pack_ext(msgpack_packer* pk, int8_t typecode, size_t l) + int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l) cdef int DEFAULT_RECURSE_LIMIT=511 diff --git a/msgpack/pack.h b/msgpack/pack.h index a71c87b..971065c 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -71,7 +71,7 @@ static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); static inline int msgpack_pack_bin(msgpack_packer* pk, size_t l); static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); -static inline int msgpack_pack_ext(msgpack_packer* pk, int8_t typecode, size_t l); +static inline int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l); static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) { From 5bc685973d133a6c47f28437d52040ee1ae0f1e5 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 7 Jan 2015 15:59:35 +0900 Subject: [PATCH 0794/1172] 0.4.3 --- ChangeLog.rst | 3 ++- README.rst | 12 +++++------- msgpack/_version.py | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index a7b9800..44e832f 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,6 @@ 0.4.3 ===== -:release date: TBD +:release date: 2015-01-07 Inconpatible Changes -------------------- @@ -12,6 +12,7 @@ Bugs fixed ---------- * Unpacker may unpack wrong uint32 value on 32bit or LLP64 environment. (#101) +* Build failed on Windows Python 2.7. 0.4.2 ===== diff --git a/README.rst b/README.rst index 705ad75..fec8aea 100644 --- a/README.rst +++ b/README.rst @@ -3,8 +3,8 @@ MessagePack for Python ======================= :author: INADA Naoki -:version: 0.4.1 -:date: 2014-02-17 +:version: 0.4.3 +:date: 2015-01-07 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png :target: https://travis-ci.org/#!/msgpack/msgpack-python @@ -18,17 +18,15 @@ writing MessagePack data. Install --------- -You can use ``pip`` or ``easy_install`` to install msgpack:: - $ easy_install msgpack-python - or +:: + $ pip install msgpack-python PyPy ^^^^^ -msgpack-python provides pure python implementation. -PyPy can use this. +msgpack-python provides pure python implementation. PyPy can use this. Windows ^^^^^^^ diff --git a/msgpack/_version.py b/msgpack/_version.py index dddfe49..367086f 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 4, 2) +version = (0, 4, 3) From c25c8d724670ceb38c0c7a75274ccfe1be18ec02 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 9 Jan 2015 03:41:52 +0900 Subject: [PATCH 0795/1172] Check extension module was compiled. --- .travis.yml | 2 +- tox.ini | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 29f1b7d..1bf6d4f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,4 +7,4 @@ install: - pip install tox cython - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx -script: "tox && MSGPACK_PUREPYTHON=x tox" +script: tox diff --git a/tox.ini b/tox.ini index 892684d..96b9fcb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,14 @@ [tox] -envlist = py26,py27,py32,py33,py34,pypy,pypy3 +envlist = {py26,py27,py32,py33,py34}-{c,pure},{pypy,pypy3}-pure + +[variants:pure] +setenv= + MSGPACK_PUREPYTHON=x [testenv] deps= pytest -commands=py.test test +commands= + c: python -c 'from msgpack import _packer, _unpacker' + pure: py.test test From ee0e435535800e51004e64d827d66bd8d30a1735 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 9 Jan 2015 04:10:25 +0900 Subject: [PATCH 0796/1172] Fix compile error. --- msgpack/pack_template.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 2879bbd..5d1088f 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -712,7 +712,7 @@ static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t /* * Ext */ -static inline int msgpack_pack_ext(msgpack_packer* x, int8_t typecode, size_t l) +static inline int msgpack_pack_ext(msgpack_packer* x, char typecode, size_t l) { if (l == 1) { unsigned char buf[2]; From 715fcac6c6e2bf95a3d074ba186bd6fb106ee312 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 9 Jan 2015 04:19:34 +0900 Subject: [PATCH 0797/1172] Fix tox --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 96b9fcb..8f10b15 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ setenv= deps= pytest +changedir=test commands= - c: python -c 'from msgpack import _packer, _unpacker' - pure: py.test test + c: python -c 'from msgpack import _packer, _unpacker' && py.test + pure: py.test From ca9768771dc63d428c3a33c046b403119e664106 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 9 Jan 2015 04:29:06 +0900 Subject: [PATCH 0798/1172] Use wheelhouse for cython --- .travis.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1bf6d4f..b9d19c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,17 @@ sudo: false +cache: + directories: + - wheelhouse + language: python python: - 2.7 install: - - pip install tox cython + - pip install wheel tox + - ls -la wheelhouse + - if [ ! -f wheelhouse/Cython-0.21.2-cp27-none-linux_x86_64.whl ] ; then pip wheel cython ; fi + - pip install wheelhouse/Cython-0.21.2-cp27-none-linux_x86_64.whl - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx script: tox From b6055ce47ea40c5dfcac96b3ff82063a849e31ce Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 9 Jan 2015 09:35:31 +0900 Subject: [PATCH 0799/1172] Fix tox.ini --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8f10b15..7971dc7 100644 --- a/tox.ini +++ b/tox.ini @@ -11,5 +11,6 @@ deps= changedir=test commands= - c: python -c 'from msgpack import _packer, _unpacker' && py.test + c: python -c 'from msgpack import _packer, _unpacker' + c: py.test pure: py.test From 3445e43d72193a95bc8ff5e8ca1dd406d2b592a4 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 9 Jan 2015 09:53:27 +0900 Subject: [PATCH 0800/1172] 0.4.4 --- ChangeLog.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 44e832f..21d2b16 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,18 @@ +0.4.4 +===== +:release date: 2015-01-09 + +Inconpatible Changes +-------------------- + +Changes +------- + +Bugs fixed +---------- + +* Fix compile error. + 0.4.3 ===== :release date: 2015-01-07 From deb8094e1d5fac491b770b6a0ec9580a665b83bc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 9 Jan 2015 09:53:44 +0900 Subject: [PATCH 0801/1172] 0.4.4 --- msgpack/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index 367086f..337c9ce 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 4, 3) +version = (0, 4, 4) From 87b493b2d84166b678aa493d12f3634e7610240e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 9 Jan 2015 09:54:30 +0900 Subject: [PATCH 0802/1172] Update README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index fec8aea..ae19984 100644 --- a/README.rst +++ b/README.rst @@ -3,8 +3,8 @@ MessagePack for Python ======================= :author: INADA Naoki -:version: 0.4.3 -:date: 2015-01-07 +:version: 0.4.4 +:date: 2015-01-09 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png :target: https://travis-ci.org/#!/msgpack/msgpack-python From 2b4a815e5a36bd8d989df35504886127cc4d7743 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Thu, 8 Jan 2015 20:37:31 -0800 Subject: [PATCH 0803/1172] README.rst: Add code-blocks For syntax highlighting --- README.rst | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index ae19984..f4268ad 100644 --- a/README.rst +++ b/README.rst @@ -52,6 +52,8 @@ It is non-string binary like Python 3's ``bytes``. To use *bin* type for packing ``bytes``, pass ``use_bin_type=True`` to packer argument. +.. code-block:: pycon + >>> import msgpack >>> packed = msgpack.packb([b'spam', u'egg'], use_bin_type=True) >>> msgpack.unpackb(packed, encoding='utf-8') @@ -62,6 +64,8 @@ binary can be unpacked by unpackers supporting msgpack-2.0. To use *ext* type, pass ``msgpack.ExtType`` object to packer. +.. code-block:: pycon + >>> import msgpack >>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy')) >>> msgpack.unpackb(packed) @@ -95,7 +99,7 @@ msgpack provides ``dumps`` and ``loads`` as alias for compatibility with ``pack`` and ``dump`` packs to file-like object. ``unpack`` and ``load`` unpacks from file-like object. -:: +.. code-block:: pycon >>> import msgpack >>> msgpack.packb([1, 2, 3]) @@ -103,7 +107,9 @@ msgpack provides ``dumps`` and ``loads`` as alias for compatibility with >>> msgpack.unpackb(_) [1, 2, 3] -``unpack`` unpacks msgpack's array to Python's list, but can unpack to tuple:: +``unpack`` unpacks msgpack's array to Python's list, but can unpack to tuple: + +.. code-block:: pycon >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False) (1, 2, 3) @@ -119,7 +125,7 @@ Streaming unpacking ``Unpacker`` is a "streaming unpacker". It unpacks multiple objects from one stream (or from bytes provided through its ``feed`` method). -:: +.. code-block:: python import msgpack from io import BytesIO @@ -141,7 +147,7 @@ Packing/unpacking of custom data type It is also possible to pack/unpack custom data types. Here is an example for ``datetime.datetime``. -:: +.. code-block:: python import datetime @@ -175,6 +181,8 @@ Extended types It is also possible to pack/unpack custom data types using the msgpack 2.0 feature. +.. code-block:: pycon + >>> import msgpack >>> import array >>> def default(obj): @@ -209,7 +217,7 @@ in a map, can be unpacked or skipped individually. Each of these methods may optionally write the packed data it reads to a callback function: -:: +.. code-block:: python from io import BytesIO From e7f505119de8eccda1476ded8087a4ea40660511 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Thu, 8 Jan 2015 22:05:27 -0800 Subject: [PATCH 0804/1172] README.rst: Add PyPI badge --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index f4268ad..1481681 100644 --- a/README.rst +++ b/README.rst @@ -8,6 +8,10 @@ MessagePack for Python .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png :target: https://travis-ci.org/#!/msgpack/msgpack-python + +.. image:: https://pypip.in/version/msgpack-python/badge.svg + :target: https://pypi.python.org/pypi/msgpack-python/ + :alt: Latest Version What's this ------------ From d14a7885c773ddf05e4431f3b43e506eb77de593 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 14 Jan 2015 10:02:04 +0900 Subject: [PATCH 0805/1172] Fix typo in ChangeLog. Fixes #121. Thanks @dmick --- ChangeLog.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 21d2b16..9f959a9 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -2,7 +2,7 @@ ===== :release date: 2015-01-09 -Inconpatible Changes +Incompatible Changes -------------------- Changes @@ -17,7 +17,7 @@ Bugs fixed ===== :release date: 2015-01-07 -Inconpatible Changes +Incompatible Changes -------------------- Changes @@ -33,7 +33,7 @@ Bugs fixed ===== :release date: 2014-03-26 -Inconpatible Changes +Incompatible Changes -------------------- Changes @@ -49,7 +49,7 @@ Bugs fixed ===== :release date: 2014-02-17 -Inconpatible Changes +Incompatible Changes -------------------- Changes @@ -67,7 +67,7 @@ Bugs fixed ===== :release date: 2013-10-21 -Inconpatible Changes +Incompatible Changes -------------------- * Raises TypeError instead of ValueError when packer receives unsupported type. @@ -81,7 +81,7 @@ Changes 0.3.0 ===== -Inconpatible Changes +Incompatible Changes -------------------- * Default value of ``use_list`` is ``True`` for now. (It was ``False`` for 0.2.x) From f40fdf523a545035d3f6ee36a31d50993feac14b Mon Sep 17 00:00:00 2001 From: Ken Dreyer Date: Fri, 23 Jan 2015 12:22:28 -0700 Subject: [PATCH 0806/1172] tests: add pytest 2.3 compatibility Adjust the skipif conditional to use the older pytest 2.3 syntax. (This allows the tests to pass with the system pytest package on RHEL 7.0, since RHEL 7.0 ships pytest 2.3.5.) --- test/test_unpack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_unpack.py b/test/test_unpack.py index 8d0d949..c0d711c 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -16,7 +16,7 @@ def test_unpack_array_header_from_file(): unpacker.unpack() -@mark.skipif(not hasattr(sys, 'getrefcount'), +@mark.skipif("not hasattr(sys, 'getrefcount') == True", reason='sys.getrefcount() is needed to pass this test') def test_unpacker_hook_refcnt(): result = [] From 75ce78dd1512460712f2600ffd927bedeeb02fbc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 23 Jun 2014 22:46:08 +0900 Subject: [PATCH 0807/1172] Add max__len option to unpacker. (fixes #97). Fix build error on 32bit environment (fixes #102). --- msgpack/_unpacker.pyx | 60 ++++++++++++++++---- msgpack/fallback.py | 127 ++++++++++++++++++++++++++++++++---------- msgpack/unpack.h | 33 +++++++++-- test/test_limits.py | 73 +++++++++++++++++++++++- 4 files changed, 246 insertions(+), 47 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 16de40f..a9770d4 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -28,6 +28,11 @@ cdef extern from "unpack.h": PyObject* ext_hook char *encoding char *unicode_errors + Py_ssize_t max_str_len + Py_ssize_t max_bin_len + Py_ssize_t max_array_len + Py_ssize_t max_map_len + Py_ssize_t max_ext_len ctypedef struct unpack_context: msgpack_user user @@ -46,10 +51,18 @@ cdef extern from "unpack.h": cdef inline init_ctx(unpack_context *ctx, object object_hook, object object_pairs_hook, object list_hook, object ext_hook, - bint use_list, char* encoding, char* unicode_errors): + bint use_list, char* encoding, char* unicode_errors, + Py_ssize_t max_str_len, Py_ssize_t max_bin_len, + Py_ssize_t max_array_len, Py_ssize_t max_map_len, + Py_ssize_t max_ext_len): unpack_init(ctx) ctx.user.use_list = use_list ctx.user.object_hook = ctx.user.list_hook = NULL + ctx.user.max_str_len = max_str_len + ctx.user.max_bin_len = max_bin_len + ctx.user.max_array_len = max_array_len + ctx.user.max_map_len = max_map_len + ctx.user.max_ext_len = max_ext_len if object_hook is not None and object_pairs_hook is not None: raise TypeError("object_pairs_hook and object_hook are mutually exclusive.") @@ -85,7 +98,12 @@ def default_read_extended_type(typecode, data): def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=1, encoding=None, unicode_errors="strict", - object_pairs_hook=None, ext_hook=ExtType): + object_pairs_hook=None, ext_hook=ExtType, + Py_ssize_t max_str_len=2147483647, # 2**32-1 + Py_ssize_t max_bin_len=2147483647, + Py_ssize_t max_array_len=2147483647, + Py_ssize_t max_map_len=2147483647, + Py_ssize_t max_ext_len=2147483647): """ Unpack packed_bytes to object. Returns an unpacked object. @@ -115,7 +133,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cerr = PyBytes_AsString(unicode_errors) init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook, - use_list, cenc, cerr) + use_list, cenc, cerr, + max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) ret = unpack_construct(&ctx, buf, buf_len, &off) if ret == 1: obj = unpack_data(&ctx) @@ -144,8 +163,7 @@ def unpack(object stream, object object_hook=None, object list_hook=None, cdef class Unpacker(object): - """ - Streaming unpacker. + """Streaming unpacker. arguments: @@ -183,6 +201,19 @@ cdef class Unpacker(object): Raises `BufferFull` exception when it is insufficient. You shoud set this parameter when unpacking data from untrasted source. + :param int max_str_len: + Limits max length of str. (default: 2**31-1) + + :param int max_bin_len: + Limits max length of bin. (default: 2**31-1) + + :param int max_array_len: + Limits max length of array. (default: 2**31-1) + + :param int max_map_len: + Limits max length of map. (default: 2**31-1) + + example of streaming deserialize from file-like object:: unpacker = Unpacker(file_like) @@ -221,7 +252,12 @@ cdef class Unpacker(object): def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1, object object_hook=None, object object_pairs_hook=None, object list_hook=None, str encoding=None, str unicode_errors='strict', int max_buffer_size=0, - object ext_hook=ExtType): + object ext_hook=ExtType, + Py_ssize_t max_str_len=2147483647, # 2**32-1 + Py_ssize_t max_bin_len=2147483647, + Py_ssize_t max_array_len=2147483647, + Py_ssize_t max_map_len=2147483647, + Py_ssize_t max_ext_len=2147483647): cdef char *cenc=NULL, cdef char *cerr=NULL @@ -265,7 +301,9 @@ cdef class Unpacker(object): cerr = PyBytes_AsString(self.unicode_errors) init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, - ext_hook, use_list, cenc, cerr) + ext_hook, use_list, cenc, cerr, + max_str_len, max_bin_len, max_array_len, + max_map_len, max_ext_len) def feed(self, object next_bytes): """Append `next_bytes` to internal buffer.""" @@ -365,7 +403,7 @@ cdef class Unpacker(object): raise ValueError("Unpack failed: error = %d" % (ret,)) def read_bytes(self, Py_ssize_t nbytes): - """read a specified number of raw bytes from the stream""" + """Read a specified number of raw bytes from the stream""" cdef size_t nread nread = min(self.buf_tail - self.buf_head, nbytes) ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread) @@ -375,8 +413,7 @@ cdef class Unpacker(object): return ret def unpack(self, object write_bytes=None): - """ - unpack one object + """Unpack one object If write_bytes is not None, it will be called with parts of the raw message as it is unpacked. @@ -386,8 +423,7 @@ cdef class Unpacker(object): return self._unpack(unpack_construct, write_bytes) def skip(self, object write_bytes=None): - """ - read and ignore one object, returning None + """Read and ignore one object, returning None If write_bytes is not None, it will be called with parts of the raw message as it is unpacked. diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 71fa7be..d1f39d1 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -102,62 +102,84 @@ def unpackb(packed, **kwargs): class Unpacker(object): - """ - Streaming unpacker. + """Streaming unpacker. - `file_like` is a file-like object having a `.read(n)` method. - When `Unpacker` is initialized with a `file_like`, `.feed()` is not - usable. + arguments: - `read_size` is used for `file_like.read(read_size)`. + :param file_like: + File-like object having `.read(n)` method. + If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. - If `use_list` is True (default), msgpack lists are deserialized to Python - lists. Otherwise they are deserialized to tuples. + :param int read_size: + Used as `file_like.read(read_size)`. (default: `min(1024**2, max_buffer_size)`) - `object_hook` is the same as in simplejson. If it is not None, it should - be callable and Unpacker calls it with a dict argument after deserializing - a map. + :param bool use_list: + If true, unpack msgpack array to Python list. + Otherwise, unpack to Python tuple. (default: True) - `object_pairs_hook` is the same as in simplejson. If it is not None, it - should be callable and Unpacker calls it with a list of key-value pairs - after deserializing a map. + :param callable object_hook: + When specified, it should be callable. + Unpacker calls it with a dict argument after unpacking msgpack map. + (See also simplejson) - `ext_hook` is callback for ext (User defined) type. It called with two - arguments: (code, bytes). default: `msgpack.ExtType` + :param callable object_pairs_hook: + When specified, it should be callable. + Unpacker calls it with a list of key-value pairs after unpacking msgpack map. + (See also simplejson) - `encoding` is the encoding used for decoding msgpack bytes. If it is - None (default), msgpack bytes are deserialized to Python bytes. + :param str encoding: + Encoding used for decoding msgpack raw. + If it is None (default), msgpack raw is deserialized to Python bytes. - `unicode_errors` is used for decoding bytes. + :param str unicode_errors: + Used for decoding msgpack raw with *encoding*. + (default: `'strict'`) - `max_buffer_size` limits the buffer size. 0 means INT_MAX (default). + :param int max_buffer_size: + Limits size of data waiting unpacked. 0 means system's INT_MAX (default). + Raises `BufferFull` exception when it is insufficient. + You shoud set this parameter when unpacking data from untrasted source. - Raises `BufferFull` exception when it is unsufficient. + :param int max_str_len: + Limits max length of str. (default: 2**31-1) - You should set this parameter when unpacking data from an untrustred source. + :param int max_bin_len: + Limits max length of bin. (default: 2**31-1) - example of streaming deserialization from file-like object:: + :param int max_array_len: + Limits max length of array. (default: 2**31-1) + + :param int max_map_len: + Limits max length of map. (default: 2**31-1) + + + example of streaming deserialize from file-like object:: unpacker = Unpacker(file_like) for o in unpacker: - do_something(o) + process(o) - example of streaming deserialization from socket:: + example of streaming deserialize from socket:: unpacker = Unpacker() - while 1: - buf = sock.recv(1024*2) + while True: + buf = sock.recv(1024**2) if not buf: break unpacker.feed(buf) for o in unpacker: - do_something(o) + process(o) """ def __init__(self, file_like=None, read_size=0, use_list=True, object_hook=None, object_pairs_hook=None, list_hook=None, encoding=None, unicode_errors='strict', max_buffer_size=0, - ext_hook=ExtType): + ext_hook=ExtType, + max_str_len=2147483647, # 2**32-1 + max_bin_len=2147483647, + max_array_len=2147483647, + max_map_len=2147483647, + max_ext_len=2147483647): if file_like is None: self._fb_feeding = True else: @@ -185,6 +207,11 @@ class Unpacker(object): self._object_hook = object_hook self._object_pairs_hook = object_pairs_hook self._ext_hook = ext_hook + self._max_str_len = max_str_len + self._max_bin_len = max_bin_len + self._max_array_len = max_array_len + self._max_map_len = max_map_len + self._max_ext_len = max_ext_len if list_hook is not None and not callable(list_hook): raise TypeError('`list_hook` is not callable') @@ -316,12 +343,18 @@ class Unpacker(object): n = b & 0b00011111 obj = self._fb_read(n, write_bytes) typ = TYPE_RAW + if n > self._max_str_len: + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) elif b & 0b11110000 == 0b10010000: n = b & 0b00001111 typ = TYPE_ARRAY + if n > self._max_array_len: + raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b & 0b11110000 == 0b10000000: n = b & 0b00001111 typ = TYPE_MAP + if n > self._max_map_len: + raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) elif b == 0xc0: obj = None elif b == 0xc2: @@ -331,26 +364,38 @@ class Unpacker(object): elif b == 0xc4: typ = TYPE_BIN n = struct.unpack("B", self._fb_read(1, write_bytes))[0] + if n > self._max_bin_len: + raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._fb_read(n, write_bytes) elif b == 0xc5: typ = TYPE_BIN n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + if n > self._max_bin_len: + raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._fb_read(n, write_bytes) elif b == 0xc6: typ = TYPE_BIN n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + if n > self._max_bin_len: + raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._fb_read(n, write_bytes) elif b == 0xc7: # ext 8 typ = TYPE_EXT L, n = struct.unpack('Bb', self._fb_read(2, write_bytes)) + if L > self._max_ext_len: + raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._fb_read(L, write_bytes) elif b == 0xc8: # ext 16 typ = TYPE_EXT L, n = struct.unpack('>Hb', self._fb_read(3, write_bytes)) + if L > self._max_ext_len: + raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._fb_read(L, write_bytes) elif b == 0xc9: # ext 32 typ = TYPE_EXT L, n = struct.unpack('>Ib', self._fb_read(5, write_bytes)) + if L > self._max_ext_len: + raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._fb_read(L, write_bytes) elif b == 0xca: obj = struct.unpack(">f", self._fb_read(4, write_bytes))[0] @@ -374,42 +419,66 @@ class Unpacker(object): obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0] elif b == 0xd4: # fixext 1 typ = TYPE_EXT + if self._max_ext_len < 1: + raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) n, obj = struct.unpack('b1s', self._fb_read(2, write_bytes)) elif b == 0xd5: # fixext 2 typ = TYPE_EXT + if self._max_ext_len < 2: + raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) n, obj = struct.unpack('b2s', self._fb_read(3, write_bytes)) elif b == 0xd6: # fixext 4 typ = TYPE_EXT + if self._max_ext_len < 4: + raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) n, obj = struct.unpack('b4s', self._fb_read(5, write_bytes)) elif b == 0xd7: # fixext 8 typ = TYPE_EXT + if self._max_ext_len < 8: + raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) n, obj = struct.unpack('b8s', self._fb_read(9, write_bytes)) elif b == 0xd8: # fixext 16 typ = TYPE_EXT + if self._max_ext_len < 16: + raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) n, obj = struct.unpack('b16s', self._fb_read(17, write_bytes)) elif b == 0xd9: typ = TYPE_RAW n = struct.unpack("B", self._fb_read(1, write_bytes))[0] + if n > self._max_str_len: + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._fb_read(n, write_bytes) elif b == 0xda: typ = TYPE_RAW n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + if n > self._max_str_len: + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._fb_read(n, write_bytes) elif b == 0xdb: typ = TYPE_RAW n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + if n > self._max_str_len: + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._fb_read(n, write_bytes) elif b == 0xdc: n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + if n > self._max_array_len: + raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) typ = TYPE_ARRAY elif b == 0xdd: n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + if n > self._max_array_len: + raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) typ = TYPE_ARRAY elif b == 0xde: n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + if n > self._max_map_len: + raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP elif b == 0xdf: n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + if n > self._max_map_len: + raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP else: raise UnpackValueError("Unknown header: 0x%x" % b) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 24045d5..5deb7cd 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -27,6 +27,7 @@ typedef struct unpack_user { PyObject *ext_hook; const char *encoding; const char *unicode_errors; + Py_ssize_t max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len; } unpack_user; typedef PyObject* msgpack_unpack_object; @@ -68,7 +69,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp if (d > LONG_MAX) { p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d); } else { - p = PyInt_FromLong((long)d); + p = PyInt_FromSize_t((size_t)d); } if (!p) return -1; @@ -132,6 +133,10 @@ static inline int unpack_callback_false(unpack_user* u, msgpack_unpack_object* o static inline int unpack_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { + if (n > u->max_array_len) { + PyErr_Format(PyExc_ValueError, "%u exceeds max_array_len(%zd)", n, u->max_array_len); + return -1; + } PyObject *p = u->use_list ? PyList_New(n) : PyTuple_New(n); if (!p) @@ -163,6 +168,10 @@ static inline int unpack_callback_array_end(unpack_user* u, msgpack_unpack_objec static inline int unpack_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { + if (n > u->max_map_len) { + PyErr_Format(PyExc_ValueError, "%u exceeds max_map_len(%zd)", n, u->max_map_len); + return -1; + } PyObject *p; if (u->has_pairs_hook) { p = PyList_New(n); // Or use tuple? @@ -210,6 +219,11 @@ static inline int unpack_callback_map_end(unpack_user* u, msgpack_unpack_object* static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { + if (l > u->max_str_len) { + PyErr_Format(PyExc_ValueError, "%u exceeds max_str_len(%zd)", l, u->max_str_len); + return -1; + } + PyObject *py; if(u->encoding) { py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors); @@ -224,6 +238,11 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { + if (l > u->max_bin_len) { + PyErr_Format(PyExc_ValueError, "%u exceeds max_bin_len(%zd)", l, u->max_bin_len); + return -1; + } + PyObject *py = PyBytes_FromStringAndSize(p, l); if (!py) return -1; @@ -232,7 +251,7 @@ static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* } static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos, - unsigned int lenght, msgpack_unpack_object* o) + unsigned int length, msgpack_unpack_object* o) { PyObject *py; int8_t typecode = (int8_t)*pos++; @@ -240,11 +259,15 @@ static inline int unpack_callback_ext(unpack_user* u, const char* base, const ch PyErr_SetString(PyExc_AssertionError, "u->ext_hook cannot be NULL"); return -1; } - // length also includes the typecode, so the actual data is lenght-1 + if (length-1 > u->max_ext_len) { + PyErr_Format(PyExc_ValueError, "%u exceeds max_ext_len(%zd)", length, u->max_ext_len); + return -1; + } + // length also includes the typecode, so the actual data is length-1 #if PY_MAJOR_VERSION == 2 - py = PyObject_CallFunction(u->ext_hook, "(is#)", typecode, pos, lenght-1); + py = PyObject_CallFunction(u->ext_hook, "(is#)", typecode, pos, length-1); #else - py = PyObject_CallFunction(u->ext_hook, "(iy#)", typecode, pos, lenght-1); + py = PyObject_CallFunction(u->ext_hook, "(iy#)", typecode, pos, length-1); #endif if (!py) return -1; diff --git a/test/test_limits.py b/test/test_limits.py index 1cfa2d6..3c1cf2a 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -3,7 +3,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals import pytest -from msgpack import packb, unpackb, Packer +from msgpack import packb, unpackb, Packer, Unpacker, ExtType def test_integer(): @@ -32,6 +32,77 @@ def test_map_header(): packer.pack_array_header(2**32) +def test_max_str_len(): + d = 'x' * 3 + packed = packb(d) + + unpacker = Unpacker(max_str_len=3, encoding='utf-8') + unpacker.feed(packed) + assert unpacker.unpack() == d + + unpacker = Unpacker(max_str_len=2, encoding='utf-8') + with pytest.raises(ValueError): + unpacker.feed(packed) + unpacker.unpack() + + +def test_max_bin_len(): + d = b'x' * 3 + packed = packb(d, use_bin_type=True) + + unpacker = Unpacker(max_bin_len=3) + unpacker.feed(packed) + assert unpacker.unpack() == d + + unpacker = Unpacker(max_bin_len=2) + with pytest.raises(ValueError): + unpacker.feed(packed) + unpacker.unpack() + + +def test_max_array_len(): + d = [1,2,3] + packed = packb(d) + + unpacker = Unpacker(max_array_len=3) + unpacker.feed(packed) + assert unpacker.unpack() == d + + unpacker = Unpacker(max_array_len=2) + with pytest.raises(ValueError): + unpacker.feed(packed) + unpacker.unpack() + + +def test_max_map_len(): + d = {1: 2, 3: 4, 5: 6} + packed = packb(d) + + unpacker = Unpacker(max_map_len=3) + unpacker.feed(packed) + assert unpacker.unpack() == d + + unpacker = Unpacker(max_map_len=2) + with pytest.raises(ValueError): + unpacker.feed(packed) + unpacker.unpack() + + +def test_max_ext_len(): + d = ExtType(42, b"abc") + packed = packb(d) + + unpacker = Unpacker(max_ext_len=3) + unpacker.feed(packed) + assert unpacker.unpack() == d + + unpacker = Unpacker(max_ext_len=2) + with pytest.raises(ValueError): + unpacker.feed(packed) + unpacker.unpack() + + + # PyPy fails following tests because of constant folding? # https://bugs.pypy.org/issue1721 #@pytest.mark.skipif(True, reason="Requires very large memory.") From 2985f4d8651982b07e2cfa7037e7a8c3530a127b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 25 Jan 2015 02:24:53 +0900 Subject: [PATCH 0808/1172] Fix error when use unicode_literal in Python 2 --- .travis.yml | 7 +++++++ msgpack/_unpacker.pyx | 10 +++++++--- tox.ini | 29 ++++++++++++++++++++++++++--- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index b9d19c1..dad7e87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,13 @@ language: python python: - 2.7 +env: + - TOXENV=py26-c,py27-c + - TOXENV=py32-c,py33-c,py34-c + - TOXENV=py26-pure,py27-pure + - TOXENV=py32-pure,py33-pure,py34-pure + - TOXENV=pypy-pure,pypy3-pure + install: - pip install wheel tox - ls -la wheelhouse diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index a9770d4..f5e7d95 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -251,7 +251,7 @@ cdef class Unpacker(object): def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1, object object_hook=None, object object_pairs_hook=None, object list_hook=None, - str encoding=None, str unicode_errors='strict', int max_buffer_size=0, + encoding=None, unicode_errors='strict', int max_buffer_size=0, object ext_hook=ExtType, Py_ssize_t max_str_len=2147483647, # 2**32-1 Py_ssize_t max_bin_len=2147483647, @@ -289,15 +289,19 @@ cdef class Unpacker(object): if encoding is not None: if isinstance(encoding, unicode): self.encoding = encoding.encode('ascii') - else: + elif isinstance(encoding, bytes): self.encoding = encoding + else: + raise TypeError("encoding should be bytes or unicode") cenc = PyBytes_AsString(self.encoding) if unicode_errors is not None: if isinstance(unicode_errors, unicode): self.unicode_errors = unicode_errors.encode('ascii') - else: + elif isinstance(unicode_errors, bytes): self.unicode_errors = unicode_errors + else: + raise TypeError("unicode_errors should be bytes or unicode") cerr = PyBytes_AsString(self.unicode_errors) init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, diff --git a/tox.ini b/tox.ini index 7971dc7..15feb51 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py26,py27,py32,py33,py34}-{c,pure},{pypy,pypy3}-pure +envlist = {py26,py27,py32,py33,py34}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86 [variants:pure] setenv= @@ -11,6 +11,29 @@ deps= changedir=test commands= - c: python -c 'from msgpack import _packer, _unpacker' - c: py.test + c,x86: python -c 'from msgpack import _packer, _unpacker' + c,x86: py.test pure: py.test + +[testenv:py27-x86] +basepython=python2.7-x86 +deps= + pytest + +changedir=test +commands= + python -c 'import sys; print(hex(sys.maxsize))' + python -c 'from msgpack import _packer, _unpacker' + py.test + +[testenv:py34-x86] +basepython=python3.4-x86 +deps= + pytest + +changedir=test +commands= + python -c 'import sys; print(hex(sys.maxsize))' + python -c 'from msgpack import _packer, _unpacker' + py.test + From 630c046bf2dcd9f77c09efb65a51ec1b3d915975 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 26 Jan 2015 00:38:36 +0900 Subject: [PATCH 0809/1172] 0.4.5 --- ChangeLog.rst | 18 ++++++++++++++++++ ROADMAP.md | 7 ------- msgpack/_version.py | 2 +- 3 files changed, 19 insertions(+), 8 deletions(-) delete mode 100644 ROADMAP.md diff --git a/ChangeLog.rst b/ChangeLog.rst index 9f959a9..797fa66 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,21 @@ +0.4.5 +===== +:release date: 2015-01-25 + +Incompatible Changes +-------------------- + +Changes +------- + +Bugs fixed +---------- + +* Fix test failure on pytest 2.3. (by @ktdreyer) +* Fix typos in ChangeLog. (Thanks to @dmick) +* Improve README.rst (by @msabramo) + + 0.4.4 ===== :release date: 2015-01-09 diff --git a/ROADMAP.md b/ROADMAP.md deleted file mode 100644 index 5245cc0..0000000 --- a/ROADMAP.md +++ /dev/null @@ -1,7 +0,0 @@ -0.2 series -========== -Improve compatibility to simplejson. - -0.3 series -========== -Add features msgpack-ruby has. diff --git a/msgpack/_version.py b/msgpack/_version.py index 337c9ce..ac12f4e 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 4, 4) +version = (0, 4, 5) From 68b029446569bdaf77518ceda99da822b0b26f21 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 26 Jan 2015 00:39:50 +0900 Subject: [PATCH 0810/1172] update README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 1481681..3c1957d 100644 --- a/README.rst +++ b/README.rst @@ -3,8 +3,8 @@ MessagePack for Python ======================= :author: INADA Naoki -:version: 0.4.4 -:date: 2015-01-09 +:version: 0.4.5 +:date: 2015-01-25 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png :target: https://travis-ci.org/#!/msgpack/msgpack-python From 83404945c06baa3124d7e20a2d4e93652ecf90c5 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Mon, 26 Jan 2015 20:31:03 +0100 Subject: [PATCH 0811/1172] Add test for issue #124 --- test/test_sequnpack.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 5d37698..518b905 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -84,3 +84,15 @@ def test_readbytes(): assert unpacker.read_bytes(3) == b'oob' assert unpacker.unpack() == ord(b'a') assert unpacker.unpack() == ord(b'r') + +def test_issue124(): + unpacker = Unpacker() + unpacker.feed('\xa1?\xa1!') + assert tuple(unpacker) == ('?', '!') + assert tuple(unpacker) == () + unpacker.feed("\xa1?\xa1") + assert tuple(unpacker) == ('?',) + assert tuple(unpacker) == () + unpacker.feed("!") + assert tuple(unpacker) == ('!',) + assert tuple(unpacker) == () From a71a24d86aca4747a954ac642a9173a05cedd257 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Mon, 26 Jan 2015 20:34:31 +0100 Subject: [PATCH 0812/1172] Fix #124 When using Unpacker as an iterator, after each yield, the internal buffer (_fb_buffer) was compacted by reallocation (done by _fb_consume). When dealing with a lot of small objects, this is very ineffecient. Thus in commit 7eb371f8278941fb2323e0c2333ed89c88ab822b the pure python fallback only reallocated the complete buffer when the iteration stops. When halfway there happens to be data missing in the buffer, we rollback the buffer to the state before this failed call, and raise an OutOfData. This rollback, done by _fb_rollback, did not consider the possibility that the buffer was *not* reallocated. This commit corrects that. --- msgpack/fallback.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index d1f39d1..bd99a15 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -195,6 +195,9 @@ class Unpacker(object): # the buffer is not "consumed" completely, for efficiency sake. # Instead, it is done sloppily. To make sure we raise BufferFull at # the correct moments, we have to keep track of how sloppy we were. + # Furthermore, when the buffer is incomplete (that is: in the case + # we raise an OutOfData) we need to rollback the buffer to the correct + # state, which _fb_slopiness records. self._fb_sloppiness = 0 self._max_buffer_size = max_buffer_size or 2**31-1 if read_size > self._max_buffer_size: @@ -283,7 +286,7 @@ class Unpacker(object): def _fb_rollback(self): self._fb_buf_i = 0 - self._fb_buf_o = 0 + self._fb_buf_o = self._fb_sloppiness def _fb_get_extradata(self): bufs = self._fb_buffers[self._fb_buf_i:] From c5d621853d305418feb3ef2616a05f5eb12440bb Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Mon, 26 Jan 2015 20:55:23 +0100 Subject: [PATCH 0813/1172] test_sequnpack: python3 literals --- test/test_sequnpack.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 518b905..45f4cc7 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -87,12 +87,12 @@ def test_readbytes(): def test_issue124(): unpacker = Unpacker() - unpacker.feed('\xa1?\xa1!') - assert tuple(unpacker) == ('?', '!') + unpacker.feed(b'\xa1?\xa1!') + assert tuple(unpacker) == (b'?', b'!') assert tuple(unpacker) == () - unpacker.feed("\xa1?\xa1") - assert tuple(unpacker) == ('?',) + unpacker.feed(b"\xa1?\xa1") + assert tuple(unpacker) == (b'?',) assert tuple(unpacker) == () - unpacker.feed("!") - assert tuple(unpacker) == ('!',) + unpacker.feed(b"!") + assert tuple(unpacker) == (b'!',) assert tuple(unpacker) == () From 4576b94b6c9c641bfa29a1d161968822d6e67d19 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 27 Jan 2015 14:04:32 +0900 Subject: [PATCH 0814/1172] fallback: Add some comment to Unpacker members. --- msgpack/fallback.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index bd99a15..235c201 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -187,10 +187,16 @@ class Unpacker(object): raise TypeError("`file_like.read` must be callable") self.file_like = file_like self._fb_feeding = False + + #: array of bytes feeded. self._fb_buffers = [] - self._fb_buf_o = 0 + #: Which buffer we currently reads self._fb_buf_i = 0 + #: Which position we currently reads + self._fb_buf_o = 0 + #: Total size of _fb_bufferes self._fb_buf_n = 0 + # When Unpacker is used as an iterable, between the calls to next(), # the buffer is not "consumed" completely, for efficiency sake. # Instead, it is done sloppily. To make sure we raise BufferFull at @@ -199,6 +205,7 @@ class Unpacker(object): # we raise an OutOfData) we need to rollback the buffer to the correct # state, which _fb_slopiness records. self._fb_sloppiness = 0 + self._max_buffer_size = max_buffer_size or 2**31-1 if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") From 9fe19cc4089467fff185399c659ffe72f2f52995 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 13 Mar 2015 03:51:14 +0900 Subject: [PATCH 0815/1172] 0.4.6 --- ChangeLog.rst | 11 +++++++++++ msgpack/_version.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 797fa66..34f4cd4 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,14 @@ +0.4.6 +===== +:release date: 2015-03-13 + +Bugs fixed +---------- + +* fallback.Unpacker: Fix Data corruption when OutOfData. + This bug only affects "Streaming unpacking." + + 0.4.5 ===== :release date: 2015-01-25 diff --git a/msgpack/_version.py b/msgpack/_version.py index ac12f4e..2c1c96c 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 4, 5) +version = (0, 4, 6) From b19e336108ed86ba344eeaccc9476244848e4dd4 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 13 Mar 2015 04:05:44 +0900 Subject: [PATCH 0816/1172] travis: Cython 0.22 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index dad7e87..ad54ee0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,8 @@ env: install: - pip install wheel tox - ls -la wheelhouse - - if [ ! -f wheelhouse/Cython-0.21.2-cp27-none-linux_x86_64.whl ] ; then pip wheel cython ; fi - - pip install wheelhouse/Cython-0.21.2-cp27-none-linux_x86_64.whl + - if [ ! -f wheelhouse/Cython-0.22-cp27-none-linux_x86_64.whl ] ; then pip wheel cython==0.22 ; fi + - pip install wheelhouse/Cython-0.22-cp27-none-linux_x86_64.whl - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx script: tox From 2dda8fc4a58dd9be0c8d6f472342fd777d92886d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 13 Mar 2015 04:18:10 +0900 Subject: [PATCH 0817/1172] travis: Build only master --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index ad54ee0..ddd5254 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,10 @@ language: python python: - 2.7 +branches: + only: + - master + env: - TOXENV=py26-c,py27-c - TOXENV=py32-c,py33-c,py34-c From b7806a6e6eceb13153df6907b744af3be0e1075e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 13 Mar 2015 04:23:04 +0900 Subject: [PATCH 0818/1172] README: Update version --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 3c1957d..8cee306 100644 --- a/README.rst +++ b/README.rst @@ -3,8 +3,8 @@ MessagePack for Python ======================= :author: INADA Naoki -:version: 0.4.5 -:date: 2015-01-25 +:version: 0.4.6 +:date: 2015-03-13 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png :target: https://travis-ci.org/#!/msgpack/msgpack-python From 2d05b40b030cb6b5da0913e72b59a91b09faccab Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Tue, 17 Mar 2015 15:02:40 -0400 Subject: [PATCH 0819/1172] Test to demonstrate that the default function isn't always called (#133) --- test/test_extension.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/test_extension.py b/test/test_extension.py index 2f85ce3..c552498 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -55,3 +55,22 @@ def test_extension_type(): s = msgpack.packb(obj, default=default) obj2 = msgpack.unpackb(s, ext_hook=ext_hook) assert obj == obj2 + +import sys +if sys.version > '3': + long = int + +def test_overriding_hooks(): + def default(obj): + if isinstance(obj, long): + return {"__type__": "long", "__data__": str(obj)} + else: + return obj + + obj = {"testval": long(1823746192837461928374619)} + refobj = {"testval": default(obj["testval"])} + refout = msgpack.packb(refobj) + assert isinstance(refout, (str, bytes)) + testout = msgpack.packb(obj, default=default) + + assert refout == testout From 10cd2d2ebf6390e844c2bf59e9efd765f9b60e40 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Tue, 17 Mar 2015 15:05:04 -0400 Subject: [PATCH 0820/1172] calling the default function upon integer overflow in the fallback routine --- msgpack/fallback.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 235c201..eb20002 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -655,6 +655,10 @@ class Packer(object): return self._buffer.write(struct.pack(">BQ", 0xcf, obj)) if -0x8000000000000000 <= obj < -0x80000000: return self._buffer.write(struct.pack(">Bq", 0xd3, obj)) + if not default_used and self._default is not None: + obj = self._default(obj) + default_used = True + continue raise PackValueError("Integer value out of range") if self._use_bin_type and isinstance(obj, bytes): n = len(obj) From 6f02d252e1dc66d67861b45c5bead8392ed822d4 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Tue, 17 Mar 2015 15:16:17 -0400 Subject: [PATCH 0821/1172] corresponding change to cython implementation --- msgpack/_packer.pyx | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index fcd20a7..7129208 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -136,12 +136,20 @@ cdef class Packer(object): elif PyLong_Check(o): # PyInt_Check(long) is True for Python 3. # Sow we should test long before int. - if o > 0: - ullval = o - ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) - else: - llval = o - ret = msgpack_pack_long_long(&self.pk, llval) + try: + if o > 0: + ullval = o + ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) + else: + llval = o + ret = msgpack_pack_long_long(&self.pk, llval) + except OverflowError, oe: + if not default_used and self._default is not None: + o = self._default(o) + default_used = True + continue + else: + raise elif PyInt_Check(o): longval = o ret = msgpack_pack_long(&self.pk, longval) From 734cb71dac66b9760cd9704667df92d0a097c1a1 Mon Sep 17 00:00:00 2001 From: tbeu Date: Sun, 22 Mar 2015 21:35:21 +0100 Subject: [PATCH 0822/1172] Update README.rst Fix typo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 8cee306..ba2c1b5 100644 --- a/README.rst +++ b/README.rst @@ -37,7 +37,7 @@ Windows When you can't use binary distribution, you need to install Visual Studio or Windows SDK on Windows. (NOTE: Visual C++ Express 2010 doesn't support -amd64. Windows SDK is recommanded way to build amd64 msgpack without any fee.) +amd64. Windows SDK is recommended way to build amd64 msgpack without any fee.) Without extension, using pure python implementation on CPython runs slowly. From 4eb4c7a9940a5424ded190c1e0b9a0bb466047fb Mon Sep 17 00:00:00 2001 From: Johannes Dollinger Date: Mon, 27 Jul 2015 20:29:43 +0200 Subject: [PATCH 0823/1172] Accept ext_hook for unpack() --- msgpack/_unpacker.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index f5e7d95..d53f724 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -147,7 +147,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=1, encoding=None, unicode_errors="strict", - object_pairs_hook=None, + object_pairs_hook=None, ext_hook=ExtType ): """ Unpack an object from `stream`. @@ -158,7 +158,7 @@ def unpack(object stream, object object_hook=None, object list_hook=None, """ return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, - encoding=encoding, unicode_errors=unicode_errors, + encoding=encoding, unicode_errors=unicode_errors, ext_hook=ext_hook ) From c3a3f9b0a5c3a380018aa42e2f81b0c6752afcf0 Mon Sep 17 00:00:00 2001 From: TW Date: Fri, 30 Oct 2015 00:36:12 +0100 Subject: [PATCH 0824/1172] fix typo in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1055a61..37729bd 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ class BuildExt(build_ext): try: return build_ext.build_extension(self, ext) except Exception as e: - print("WARNING: Failed to compile extensiom modules.") + print("WARNING: Failed to compile extension modules.") print("msgpack uses fallback pure python implementation.") print(e) From 672b220a3ff0c84eec1c8b39abcdedcd1e4cb329 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 7 Nov 2015 13:17:28 +0900 Subject: [PATCH 0825/1172] remove unused bat file --- upload_windows.bat | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 upload_windows.bat diff --git a/upload_windows.bat b/upload_windows.bat deleted file mode 100644 index 5cd9a7c..0000000 --- a/upload_windows.bat +++ /dev/null @@ -1,4 +0,0 @@ -c:\Python27\python setup.py bdist_egg bdist_wininst upload -c:\Python33\python setup.py bdist_egg bdist_wininst upload -c:\Python27_amd64\python setup.py bdist_egg bdist_wininst upload -c:\Python33_amd64\python setup.py bdist_egg bdist_wininst upload From 52a38c6e9d6cb5206732231c50d1b9a5bd0be586 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 7 Nov 2015 14:26:14 +0900 Subject: [PATCH 0826/1172] remove unused bat file --- build_windows.bat | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 build_windows.bat diff --git a/build_windows.bat b/build_windows.bat deleted file mode 100644 index a71c0e0..0000000 --- a/build_windows.bat +++ /dev/null @@ -1,24 +0,0 @@ -set MSSdk=1 -set DISTUTILS_USE_SDK=1 - -rem Python27 x86 -rem call "C:\Program Files\Microsoft SDKs\Windows\v6.1\Bin\SetEnv.cmd" /Release /x86 /xp -call "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars32.bat" -c:\Python27\python setup.py build_ext -f build install -pause - -rem Python27 amd64 -rem call "C:\Program Files\Microsoft SDKs\Windows\v6.1\Bin\SetEnv.cmd" /Release /x64 /xp -call "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars64.bat" -c:\Python27_amd64\python setup.py build_ext -f build install -pause - -rem Python33 x86 -call "C:\Program Files\Microsoft SDKs\Windows\v7.1\bin\SetEnv.cmd" /Release /x86 /xp -c:\Python33\python setup.py build_ext -f build install -pause - -rem Python33 amd64 -call "C:\Program Files\Microsoft SDKs\Windows\v7.1\bin\SetEnv.cmd" /Release /x64 /xp -c:\Python33_amd64\python setup.py build_ext -f build install -pause From c102e6cee58df543d352e36f6d2d0bdd595e1063 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 7 Nov 2015 14:30:05 +0900 Subject: [PATCH 0827/1172] executable setup.py --- setup.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 setup.py diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 From ab359e333044fc89559b946cfec8efb83c2c6122 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 7 Nov 2015 16:45:30 +0900 Subject: [PATCH 0828/1172] Update travis setting --- .travis.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index ddd5254..7695184 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,5 @@ sudo: false -cache: - directories: - - wheelhouse - +cache: pip language: python python: - 2.7 @@ -19,10 +16,8 @@ env: - TOXENV=pypy-pure,pypy3-pure install: - - pip install wheel tox - - ls -la wheelhouse - - if [ ! -f wheelhouse/Cython-0.22-cp27-none-linux_x86_64.whl ] ; then pip wheel cython==0.22 ; fi - - pip install wheelhouse/Cython-0.22-cp27-none-linux_x86_64.whl + - pip install tox + - pip install cython --install-option=--cython-with-refnanny --install-option=--no-cython-compile - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx script: tox From e9ab4d8824fddd070bac7cedca332130bf2028b0 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 7 Nov 2015 16:30:18 +0900 Subject: [PATCH 0829/1172] Fix warnings fixes #146 --- msgpack/_unpacker.pyx | 24 ++++++++++++------------ msgpack/unpack.h | 2 +- msgpack/unpack_template.h | 10 +++++----- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index d53f724..997979c 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -37,10 +37,10 @@ cdef extern from "unpack.h": ctypedef struct unpack_context: msgpack_user user PyObject* obj - size_t count + Py_ssize_t count ctypedef int (*execute_fn)(unpack_context* ctx, const char* data, - size_t len, size_t* off) except? -1 + Py_ssize_t len, Py_ssize_t* off) except? -1 execute_fn unpack_construct execute_fn unpack_skip execute_fn read_array_header @@ -112,7 +112,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, See :class:`Unpacker` for options. """ cdef unpack_context ctx - cdef size_t off = 0 + cdef Py_ssize_t off = 0 cdef int ret cdef char* buf @@ -142,7 +142,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) return obj else: - raise UnpackValueError("Unpack failed: error = %d" % (ret,)) + raise UnpackValueError("Unpack failed: error = %s" % (ret,)) def unpack(object stream, object object_hook=None, object list_hook=None, @@ -233,14 +233,14 @@ cdef class Unpacker(object): """ cdef unpack_context ctx cdef char* buf - cdef size_t buf_size, buf_head, buf_tail + cdef Py_ssize_t buf_size, buf_head, buf_tail cdef object file_like cdef object file_like_read cdef Py_ssize_t read_size # To maintain refcnt. cdef object object_hook, object_pairs_hook, list_hook, ext_hook cdef object encoding, unicode_errors - cdef size_t max_buffer_size + cdef Py_ssize_t max_buffer_size def __cinit__(self): self.buf = NULL @@ -325,10 +325,10 @@ cdef class Unpacker(object): cdef: char* buf = self.buf char* new_buf - size_t head = self.buf_head - size_t tail = self.buf_tail - size_t buf_size = self.buf_size - size_t new_size + Py_ssize_t head = self.buf_head + Py_ssize_t tail = self.buf_tail + Py_ssize_t buf_size = self.buf_size + Py_ssize_t new_size if tail + _buf_len > buf_size: if ((tail - head) + _buf_len) <= buf_size: @@ -374,7 +374,7 @@ cdef class Unpacker(object): cdef object _unpack(self, execute_fn execute, object write_bytes, bint iter=0): cdef int ret cdef object obj - cdef size_t prev_head + cdef Py_ssize_t prev_head if self.buf_head >= self.buf_tail and self.file_like is not None: self.read_from_file() @@ -408,7 +408,7 @@ cdef class Unpacker(object): def read_bytes(self, Py_ssize_t nbytes): """Read a specified number of raw bytes from the stream""" - cdef size_t nread + cdef Py_ssize_t nread nread = min(self.buf_tail - self.buf_head, nbytes) ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread) self.buf_head += nread diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 5deb7cd..297bc93 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -33,7 +33,7 @@ typedef struct unpack_user { typedef PyObject* msgpack_unpack_object; struct unpack_context; typedef struct unpack_context unpack_context; -typedef int (*execute_fn)(unpack_context *ctx, const char* data, size_t len, size_t* off); +typedef int (*execute_fn)(unpack_context *ctx, const char* data, Py_ssize_t len, Py_ssize_t* off); static inline msgpack_unpack_object unpack_callback_root(unpack_user* u) { diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index d34eced..5b389b8 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -24,8 +24,8 @@ typedef struct unpack_stack { PyObject* obj; - size_t size; - size_t count; + Py_ssize_t size; + Py_ssize_t count; unsigned int ct; PyObject* map_key; } unpack_stack; @@ -72,7 +72,7 @@ static inline PyObject* unpack_data(unpack_context* ctx) template -static inline int unpack_execute(unpack_context* ctx, const char* data, size_t len, size_t* off) +static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) { assert(len >= *off); @@ -89,7 +89,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l */ unpack_user* user = &ctx->user; - PyObject* obj; + PyObject* obj = NULL; unpack_stack* c = NULL; int ret; @@ -409,7 +409,7 @@ _end: #undef start_container template -static inline int unpack_container_header(unpack_context* ctx, const char* data, size_t len, size_t* off) +static inline int unpack_container_header(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) { assert(len >= *off); uint32_t size; From 35a69ac9c2fbf6b68b970352791f6d98fbd74963 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 8 Nov 2015 12:38:38 +0900 Subject: [PATCH 0830/1172] Decrease refcnt when error happend while unpacking Fixes #152 --- msgpack/_unpacker.pyx | 5 +++-- msgpack/unpack_template.h | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 997979c..d359e57 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -47,6 +47,7 @@ cdef extern from "unpack.h": execute_fn read_map_header void unpack_init(unpack_context* ctx) object unpack_data(unpack_context* ctx) + void unpack_clear(unpack_context* ctx) cdef inline init_ctx(unpack_context *ctx, object object_hook, object object_pairs_hook, @@ -141,8 +142,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, if off < buf_len: raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) return obj - else: - raise UnpackValueError("Unpack failed: error = %s" % (ret,)) + unpack_clear(&ctx) + raise UnpackValueError("Unpack failed: error = %d" % (ret,)) def unpack(object stream, object object_hook=None, object list_hook=None, diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 5b389b8..6b83d3e 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -70,6 +70,10 @@ static inline PyObject* unpack_data(unpack_context* ctx) return (ctx)->stack[0].obj; } +static inline PyObject* unpack_clear(unpack_context *ctx) +{ + Py_CLEAR(ctx->stack[0].obj); +} template static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) From 02611afd5f32aa173f9ba8a777ba49fa6b4f67c5 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 8 Nov 2015 17:29:09 +0900 Subject: [PATCH 0831/1172] Drpo pypip.in badge It downs so long --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index ba2c1b5..baa825a 100644 --- a/README.rst +++ b/README.rst @@ -9,9 +9,6 @@ MessagePack for Python .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png :target: https://travis-ci.org/#!/msgpack/msgpack-python -.. image:: https://pypip.in/version/msgpack-python/badge.svg - :target: https://pypi.python.org/pypi/msgpack-python/ - :alt: Latest Version What's this ------------ From 6f208abbc7c9567145bf8c17bc41ba7c30ade0be Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 8 Nov 2015 17:34:52 +0900 Subject: [PATCH 0832/1172] Update Windows compiler information --- README.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index baa825a..456de2b 100644 --- a/README.rst +++ b/README.rst @@ -33,11 +33,15 @@ Windows ^^^^^^^ When you can't use binary distribution, you need to install Visual Studio -or Windows SDK on Windows. (NOTE: Visual C++ Express 2010 doesn't support -amd64. Windows SDK is recommended way to build amd64 msgpack without any fee.) - +or Windows SDK on Windows. Without extension, using pure python implementation on CPython runs slowly. +For Python 2.7, [Microsoft Visual C++ Compiler for Python 2.7](https://www.microsoft.com/en-us/download/details.aspx?id=44266) +is recommended solution. + +For Python 3.5, [Microsoft Visual Studio 2015](https://www.visualstudio.com/en-us/products/vs-2015-product-editions.aspx) +Community Edition or Express Edition can be used to build extension module. + Notes ----- From 53fcd9b9df1c1f84719dfc9744217a1c3a025ef5 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 8 Nov 2015 19:37:40 +0900 Subject: [PATCH 0833/1172] Update gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 1bd68b4..70f5746 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ dist/* msgpack/__version__.py msgpack/*.cpp *.egg-info +/venv +/tags +/docs/_build From e601ef4c23c6bfa64f86dcb91d28f370e77b17bc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 9 Nov 2015 00:43:52 +0900 Subject: [PATCH 0834/1172] Remove `msgpack 2.0` from README There are no versio in spec. --- README.rst | 109 +++++++++++++++++++++++++++++------------------------ 1 file changed, 59 insertions(+), 50 deletions(-) diff --git a/README.rst b/README.rst index 456de2b..30453bd 100644 --- a/README.rst +++ b/README.rst @@ -42,54 +42,6 @@ is recommended solution. For Python 3.5, [Microsoft Visual Studio 2015](https://www.visualstudio.com/en-us/products/vs-2015-product-editions.aspx) Community Edition or Express Edition can be used to build extension module. -Notes ------ - -Note for msgpack 2.0 support -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -msgpack 2.0 adds two types: *bin* and *ext*. - -*raw* was bytes or string type like Python 2's ``str``. -To distinguish string and bytes, msgpack 2.0 adds *bin*. -It is non-string binary like Python 3's ``bytes``. - -To use *bin* type for packing ``bytes``, pass ``use_bin_type=True`` to -packer argument. - -.. code-block:: pycon - - >>> import msgpack - >>> packed = msgpack.packb([b'spam', u'egg'], use_bin_type=True) - >>> msgpack.unpackb(packed, encoding='utf-8') - ['spam', u'egg'] - -You shoud use it carefully. When you use ``use_bin_type=True``, packed -binary can be unpacked by unpackers supporting msgpack-2.0. - -To use *ext* type, pass ``msgpack.ExtType`` object to packer. - -.. code-block:: pycon - - >>> import msgpack - >>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy')) - >>> msgpack.unpackb(packed) - ExtType(code=42, data='xyzzy') - -You can use it with ``default`` and ``ext_hook``. See below. - -Note for msgpack 0.2.x users -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The msgpack 0.3 have some incompatible changes. - -The default value of ``use_list`` keyword argument is ``True`` from 0.3. -You should pass the argument explicitly for backward compatibility. - -`Unpacker.unpack()` and some unpack methods now raises `OutOfData` -instead of `StopIteration`. -`StopIteration` is used for iterator protocol only. - How to use ----------- @@ -184,7 +136,7 @@ key-value pairs. Extended types ^^^^^^^^^^^^^^^ -It is also possible to pack/unpack custom data types using the msgpack 2.0 feature. +It is also possible to pack/unpack custom data types using the **ext** type. .. code-block:: pycon @@ -238,6 +190,58 @@ callback function: unpacker.skip(bytestream.write) worker.send(bytestream.getvalue()) + +Notes +----- + +string and binary type +^^^^^^^^^^^^^^^^^^^^^^ + +In old days, msgpack doesn't distinguish string and binary types like Python 1. +The type for represent string and binary types is named **raw**. + +msgpack can distinguish string and binary type for now. But it is not like Python 2. +Python 2 added unicode string. But msgpack renamed **raw** to **str** and added **bin** type. +It is because keep compatibility with data created by old libs. **raw** was used for text more than binary. + +Currently, while msgpack-python supports new **bin** type, default setting doesn't use it and +decodes **raw** as `bytes` instead of `unicode` (`str` in Python 3). + +You can change this by using `use_bin_type=True` option in Packer and `encoding="utf-8"` option in Unpacker. + +.. code-block:: pycon + + >>> import msgpack + >>> packed = msgpack.packb([b'spam', u'egg'], use_bin_type=True) + >>> msgpack.unpackb(packed, encoding='utf-8') + ['spam', u'egg'] + +ext type +^^^^^^^^ + +To use **ext** type, pass ``msgpack.ExtType`` object to packer. + +.. code-block:: pycon + + >>> import msgpack + >>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy')) + >>> msgpack.unpackb(packed) + ExtType(code=42, data='xyzzy') + +You can use it with ``default`` and ``ext_hook``. See below. + +Note for msgpack-python 0.2.x users +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The msgpack-python 0.3 have some incompatible changes. + +The default value of ``use_list`` keyword argument is ``True`` from 0.3. +You should pass the argument explicitly for backward compatibility. + +`Unpacker.unpack()` and some unpack methods now raises `OutOfData` +instead of `StopIteration`. +`StopIteration` is used for iterator protocol only. + Note about performance ------------------------ @@ -259,12 +263,17 @@ Python's dict can't use list as key and MessagePack allows array for key of mapp Another way to unpacking such object is using ``object_pairs_hook``. +Development +------------ + Test ----- +^^^^ + MessagePack uses `pytest` for testing. Run test with following command: $ py.test + .. vim: filetype=rst From 8aadc5c380d0d135273729333ace91d3f689702d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 9 Nov 2015 00:50:07 +0900 Subject: [PATCH 0835/1172] readme: Fix markup --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 30453bd..d32ec1d 100644 --- a/README.rst +++ b/README.rst @@ -36,10 +36,10 @@ When you can't use binary distribution, you need to install Visual Studio or Windows SDK on Windows. Without extension, using pure python implementation on CPython runs slowly. -For Python 2.7, [Microsoft Visual C++ Compiler for Python 2.7](https://www.microsoft.com/en-us/download/details.aspx?id=44266) +For Python 2.7, `Microsoft Visual C++ Compiler for Python 2.7 `_ is recommended solution. -For Python 3.5, [Microsoft Visual Studio 2015](https://www.visualstudio.com/en-us/products/vs-2015-product-editions.aspx) +For Python 3.5, `Microsoft Visual Studio 2015 `_ Community Edition or Express Edition can be used to build extension module. From 3cef27b69b250f9abf126c85578a821c2d21e72e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 9 Nov 2015 00:54:06 +0900 Subject: [PATCH 0836/1172] Update ChangeLog --- ChangeLog.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 34f4cd4..31a64d9 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,20 @@ +0.4.7 +===== +:release date: TBD + +Bugs fixed +---------- + +* Memory leak when unpack is failed + +Changes +------- + +* Reduce compiler warnings while building extension module +* unpack() now accepts ext_hook argument like Unpacker and unpackb() +* Update Cython version to 0.23.4 + + 0.4.6 ===== :release date: 2015-03-13 From de3c2b99f78d134c326bc375f19f54b7c851797a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 9 Nov 2015 01:50:40 +0900 Subject: [PATCH 0837/1172] refactor C code fixes #137 --- msgpack/pack.h | 34 ---------------------------------- msgpack/unpack.h | 4 ++-- msgpack/unpack_template.h | 2 +- 3 files changed, 3 insertions(+), 37 deletions(-) diff --git a/msgpack/pack.h b/msgpack/pack.h index 971065c..a75bdb0 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -39,40 +39,6 @@ typedef struct msgpack_packer { typedef struct Packer Packer; -static inline int msgpack_pack_int(msgpack_packer* pk, int d); -static inline int msgpack_pack_long(msgpack_packer* pk, long d); -static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d); -static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); -static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); -static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); -//static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); - -static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); -static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); -static inline int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); -static inline int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); -static inline int msgpack_pack_int8(msgpack_packer* pk, int8_t d); -static inline int msgpack_pack_int16(msgpack_packer* pk, int16_t d); -static inline int msgpack_pack_int32(msgpack_packer* pk, int32_t d); -static inline int msgpack_pack_int64(msgpack_packer* pk, int64_t d); - -static inline int msgpack_pack_float(msgpack_packer* pk, float d); -static inline int msgpack_pack_double(msgpack_packer* pk, double d); - -static inline int msgpack_pack_nil(msgpack_packer* pk); -static inline int msgpack_pack_true(msgpack_packer* pk); -static inline int msgpack_pack_false(msgpack_packer* pk); - -static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n); - -static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n); - -static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); -static inline int msgpack_pack_bin(msgpack_packer* pk, size_t l); -static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); - -static inline int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l); - static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) { char* buf = pk->buf; diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 297bc93..92f4f11 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -69,7 +69,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp if (d > LONG_MAX) { p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d); } else { - p = PyInt_FromSize_t((size_t)d); + p = PyInt_FromLong((long)d); } if (!p) return -1; @@ -100,7 +100,7 @@ static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpac { PyObject *p; if (d > LONG_MAX || d < LONG_MIN) { - p = PyLong_FromLongLong((unsigned PY_LONG_LONG)d); + p = PyLong_FromLongLong((PY_LONG_LONG)d); } else { p = PyInt_FromLong((long)d); } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 6b83d3e..e1e08fe 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -70,7 +70,7 @@ static inline PyObject* unpack_data(unpack_context* ctx) return (ctx)->stack[0].obj; } -static inline PyObject* unpack_clear(unpack_context *ctx) +static inline void unpack_clear(unpack_context *ctx) { Py_CLEAR(ctx->stack[0].obj); } From f7d3715f2cedb09babbcdd1950ecc50f4d673fbe Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 9 Nov 2015 02:00:48 +0900 Subject: [PATCH 0838/1172] Add missing params to unpack() --- msgpack/_unpacker.pyx | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index d359e57..36cb78e 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -148,7 +148,12 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=1, encoding=None, unicode_errors="strict", - object_pairs_hook=None, ext_hook=ExtType + object_pairs_hook=None, ext_hook=ExtType, + Py_ssize_t max_str_len=2147483647, # 2**32-1 + Py_ssize_t max_bin_len=2147483647, + Py_ssize_t max_array_len=2147483647, + Py_ssize_t max_map_len=2147483647, + Py_ssize_t max_ext_len=2147483647): ): """ Unpack an object from `stream`. @@ -159,7 +164,12 @@ def unpack(object stream, object object_hook=None, object list_hook=None, """ return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, - encoding=encoding, unicode_errors=unicode_errors, ext_hook=ext_hook + encoding=encoding, unicode_errors=unicode_errors, ext_hook=ext_hook, + max_str_len=max_str_len, + max_bin_len=max_bin_len, + max_array_len=max_array_len, + max_map_len=max_map_len, + max_ext_len=max_ext_len, ) From 7d900371c8f13fa64f06aaf336b6ae65c705bf2c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 9 Nov 2015 02:09:39 +0900 Subject: [PATCH 0839/1172] Fix compile error --- msgpack/_unpacker.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 36cb78e..aec3b7d 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -154,7 +154,6 @@ def unpack(object stream, object object_hook=None, object list_hook=None, Py_ssize_t max_array_len=2147483647, Py_ssize_t max_map_len=2147483647, Py_ssize_t max_ext_len=2147483647): - ): """ Unpack an object from `stream`. From a1317b604f7eb83575c4a0221db028c1929e8026 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 9 Nov 2015 02:34:17 +0900 Subject: [PATCH 0840/1172] refactor --- msgpack/_packer.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 7129208..0a6513c 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -135,7 +135,7 @@ cdef class Packer(object): ret = msgpack_pack_false(&self.pk) elif PyLong_Check(o): # PyInt_Check(long) is True for Python 3. - # Sow we should test long before int. + # So we should test long before int. try: if o > 0: ullval = o @@ -143,7 +143,7 @@ cdef class Packer(object): else: llval = o ret = msgpack_pack_long_long(&self.pk, llval) - except OverflowError, oe: + except OverflowError as oe: if not default_used and self._default is not None: o = self._default(o) default_used = True From 29266b024ec82c281fa63bf2b73622088a8c188c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 9 Nov 2015 02:34:28 +0900 Subject: [PATCH 0841/1172] Update ChangeLog --- ChangeLog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 31a64d9..35535b4 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -13,6 +13,7 @@ Changes * Reduce compiler warnings while building extension module * unpack() now accepts ext_hook argument like Unpacker and unpackb() * Update Cython version to 0.23.4 +* default function is called when integer overflow 0.4.6 From cbdf3c339a2c4c6b40a7e75dd96e457a024937f6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 10 Nov 2015 03:30:11 +0900 Subject: [PATCH 0842/1172] s/precise_mode/strict_types/ --- msgpack/_packer.pyx | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index c015fcb..50d19ff 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -55,7 +55,7 @@ cdef class Packer(object): Convert unicode to bytes with this encoding. (default: 'utf-8') :param str unicode_errors: Error handler for encoding unicode. (default: 'strict') - :param bool precise_mode: + :param bool strict_types: If set to true, types will be checked to be exact. Derived classes from serializeable types will not be serialized and will be treated as unsupported type and forwarded to default. @@ -77,7 +77,7 @@ cdef class Packer(object): cdef object _berrors cdef char *encoding cdef char *unicode_errors - cdef bint precise_mode + cdef bint strict_types cdef bool use_float cdef bint autoreset @@ -91,11 +91,11 @@ cdef class Packer(object): def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, bint autoreset=1, bint use_bin_type=0, - bint precise_mode=0): + bint strict_types=0): """ """ self.use_float = use_single_float - self.precise_mode = precise_mode + self.strict_types = strict_types self.autoreset = autoreset self.pk.use_bin_type = use_bin_type if default is not None: @@ -131,7 +131,7 @@ cdef class Packer(object): cdef dict d cdef size_t L cdef int default_used = 0 - cdef bint precise = self.precise_mode + cdef bint strict_types = self.strict_types if nest_limit < 0: raise PackValueError("recursion limit exceeded.") @@ -139,12 +139,12 @@ cdef class Packer(object): while True: if o is None: ret = msgpack_pack_nil(&self.pk) - elif PyBool_Check(o) if precise else isinstance(o, bool): + elif PyBool_Check(o) if strict_types else isinstance(o, bool): if o: ret = msgpack_pack_true(&self.pk) else: ret = msgpack_pack_false(&self.pk) - elif PyLong_CheckExact(o) if precise else PyLong_Check(o): + elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o): # PyInt_Check(long) is True for Python 3. # So we should test long before int. try: @@ -161,17 +161,17 @@ cdef class Packer(object): continue else: raise - elif PyInt_CheckExact(o) if precise else PyInt_Check(o): + elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o): longval = o ret = msgpack_pack_long(&self.pk, longval) - elif PyFloat_CheckExact(o) if precise else PyFloat_Check(o): + elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): if self.use_float: fval = o ret = msgpack_pack_float(&self.pk, fval) else: dval = o ret = msgpack_pack_double(&self.pk, dval) - elif PyBytes_CheckExact(o) if precise else PyBytes_Check(o): + elif PyBytes_CheckExact(o) if strict_types else PyBytes_Check(o): L = len(o) if L > (2**32)-1: raise ValueError("bytes is too large") @@ -179,7 +179,7 @@ cdef class Packer(object): ret = msgpack_pack_bin(&self.pk, L) if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyUnicode_CheckExact(o) if precise else PyUnicode_Check(o): + elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): if not self.encoding: raise TypeError("Can't encode unicode string: no encoding is specified") o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) @@ -202,7 +202,7 @@ cdef class Packer(object): if ret != 0: break ret = self._pack(v, nest_limit-1) if ret != 0: break - elif not precise and PyDict_Check(o): + elif not strict_types and PyDict_Check(o): L = len(o) if L > (2**32)-1: raise ValueError("dict is too large") @@ -222,7 +222,7 @@ cdef class Packer(object): raise ValueError("EXT data is too large") ret = msgpack_pack_ext(&self.pk, longval, L) ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyList_CheckExact(o) if precise else (PyTuple_Check(o) or PyList_Check(o)): + elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): L = len(o) if L > (2**32)-1: raise ValueError("list is too large") From 1032ef9bf2baef73b04f209181e42978ab4c71fe Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 10 Nov 2015 03:33:50 +0900 Subject: [PATCH 0843/1172] fallback unpacker: precise => strict --- msgpack/fallback.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 0e37f74..cffecca 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -69,6 +69,13 @@ TYPE_EXT = 5 DEFAULT_RECURSE_LIMIT = 511 +def _check_type_strict(obj, t, type=type, tuple=tuple): + if type(t) is tuple: + return type(obj) in t + else: + return type(obj) is t + + def unpack(stream, **kwargs): """ Unpack an object from `stream`. @@ -601,7 +608,7 @@ class Packer(object): Convert unicode to bytes with this encoding. (default: 'utf-8') :param str unicode_errors: Error handler for encoding unicode. (default: 'strict') - :param bool precise_mode: + :param bool strict_types: If set to true, types will be checked to be exact. Derived classes from serializeable types will not be serialized and will be treated as unsupported type and forwarded to default. @@ -618,9 +625,9 @@ class Packer(object): It also enable str8 type for unicode. """ def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', - precise_mode=False, use_single_float=False, autoreset=True, + strict_types=False, use_single_float=False, autoreset=True, use_bin_type=False): - self._precise_mode = precise_mode + self._strict_types = strict_types self._use_float = use_single_float self._autoreset = autoreset self._use_bin_type = use_bin_type @@ -632,17 +639,11 @@ class Packer(object): raise TypeError("default must be callable") self._default = default - def _check_precise(obj, t, type=type, tuple=tuple): - if type(t) is tuple: - return type(obj) in t - else: - return type(obj) is t - def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, - check=isinstance, check_precise=_check_precise): + check=isinstance, check_type_strict=_check_type_strict): default_used = False - if self._precise_mode: - check = check_precise + if self._strict_types: + check = check_type_strict list_types = list else: list_types = (list, tuple) From 9b673279d36468e3334b513b5e86d40cba4c4acc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 10 Nov 2015 03:37:54 +0900 Subject: [PATCH 0844/1172] strict_types should be last argument --- msgpack/_packer.pyx | 14 +++++++------- msgpack/fallback.py | 18 +++++++++--------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 50d19ff..c8d4fd1 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -55,13 +55,6 @@ cdef class Packer(object): Convert unicode to bytes with this encoding. (default: 'utf-8') :param str unicode_errors: Error handler for encoding unicode. (default: 'strict') - :param bool strict_types: - If set to true, types will be checked to be exact. Derived classes - from serializeable types will not be serialized and will be - treated as unsupported type and forwarded to default. - Additionally tuples will not be serialized as lists. - This is useful when trying to implement accurate serialization - for python types. :param bool use_single_float: Use single precision float type for float. (default: False) :param bool autoreset: @@ -70,6 +63,13 @@ cdef class Packer(object): :param bool use_bin_type: Use bin type introduced in msgpack spec 2.0 for bytes. It also enable str8 type for unicode. + :param bool strict_types: + If set to true, types will be checked to be exact. Derived classes + from serializeable types will not be serialized and will be + treated as unsupported type and forwarded to default. + Additionally tuples will not be serialized as lists. + This is useful when trying to implement accurate serialization + for python types. """ cdef msgpack_packer pk cdef object _default diff --git a/msgpack/fallback.py b/msgpack/fallback.py index cffecca..11280ed 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -608,13 +608,6 @@ class Packer(object): Convert unicode to bytes with this encoding. (default: 'utf-8') :param str unicode_errors: Error handler for encoding unicode. (default: 'strict') - :param bool strict_types: - If set to true, types will be checked to be exact. Derived classes - from serializeable types will not be serialized and will be - treated as unsupported type and forwarded to default. - Additionally tuples will not be serialized as lists. - This is useful when trying to implement accurate serialization - for python types. :param bool use_single_float: Use single precision float type for float. (default: False) :param bool autoreset: @@ -623,10 +616,17 @@ class Packer(object): :param bool use_bin_type: Use bin type introduced in msgpack spec 2.0 for bytes. It also enable str8 type for unicode. + :param bool strict_types: + If set to true, types will be checked to be exact. Derived classes + from serializeable types will not be serialized and will be + treated as unsupported type and forwarded to default. + Additionally tuples will not be serialized as lists. + This is useful when trying to implement accurate serialization + for python types. """ def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', - strict_types=False, use_single_float=False, autoreset=True, - use_bin_type=False): + use_single_float=False, autoreset=True, use_bin_type=False, + strict_types=False): self._strict_types = strict_types self._use_float = use_single_float self._autoreset = autoreset From 628c5191873148a8372bae4ee99454ad13b7b492 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 10 Nov 2015 03:41:09 +0900 Subject: [PATCH 0845/1172] strict type check for ext type --- msgpack/_packer.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index c8d4fd1..7c1e53d 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -213,7 +213,7 @@ cdef class Packer(object): if ret != 0: break ret = self._pack(v, nest_limit-1) if ret != 0: break - elif isinstance(o, ExtType): + elif type(o) is ExtType if strict_types else isinstance(o, ExtType): # This should be before Tuple because ExtType is namedtuple. longval = o.code rawval = o.data From 6f38bf7dd4c45523c3b0e9e8af0e69d5af35708b Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 17 Nov 2015 15:31:36 +0200 Subject: [PATCH 0846/1172] Added python 3.5 to tox.ini. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 15feb51..56ded49 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py26,py27,py32,py33,py34}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86 +envlist = {py26,py27,py32,py33,py34,py35}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86 [variants:pure] setenv= From 4d9684db0a7b2187f54f7a81a4ea64e981fa11ca Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 17 Nov 2015 15:32:34 +0200 Subject: [PATCH 0847/1172] Added Python 3.5 to the build matrix. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7695184..22f19dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,9 +10,9 @@ branches: env: - TOXENV=py26-c,py27-c - - TOXENV=py32-c,py33-c,py34-c + - TOXENV=py32-c,py33-c,py34-c,py35-c - TOXENV=py26-pure,py27-pure - - TOXENV=py32-pure,py33-pure,py34-pure + - TOXENV=py32-pure,py33-pure,py34-pure,py35-pure - TOXENV=pypy-pure,pypy3-pure install: From 81177caff7ddc9b193da9fcfd27fcd6807da7fe1 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 17 Nov 2015 16:57:25 +0200 Subject: [PATCH 0848/1172] Run the build with 3.5 since it's still not available by default in travis. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 22f19dc..7a80cdf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ sudo: false cache: pip language: python python: - - 2.7 + - 3.5 branches: only: From e4aa43d76935ac44aecccecd30fb51d00854ffac Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Tue, 17 Nov 2015 17:08:04 +0200 Subject: [PATCH 0849/1172] Travis will now cache dependencies despite having a custom install step. --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7695184..bc0efba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ sudo: false -cache: pip +cache: + directories: + - $HOME/.cache/pip language: python python: - 2.7 From 9c6584ee101d300fdf0621cac44c3dcb9df817ba Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 9 Dec 2015 13:53:42 +0100 Subject: [PATCH 0850/1172] fix typos --- msgpack/_unpacker.pyx | 2 +- msgpack/fallback.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index d53f724..b98957e 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -199,7 +199,7 @@ cdef class Unpacker(object): :param int max_buffer_size: Limits size of data waiting unpacked. 0 means system's INT_MAX (default). Raises `BufferFull` exception when it is insufficient. - You shoud set this parameter when unpacking data from untrasted source. + You shoud set this parameter when unpacking data from untrusted source. :param int max_str_len: Limits max length of str. (default: 2**31-1) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 235c201..5b5085e 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -138,7 +138,7 @@ class Unpacker(object): :param int max_buffer_size: Limits size of data waiting unpacked. 0 means system's INT_MAX (default). Raises `BufferFull` exception when it is insufficient. - You shoud set this parameter when unpacking data from untrasted source. + You shoud set this parameter when unpacking data from untrusted source. :param int max_str_len: Limits max length of str. (default: 2**31-1) From 83424bd7b3c4a417fe1dca4523adc177cee1bef9 Mon Sep 17 00:00:00 2001 From: Sadayuki Furuhashi Date: Mon, 11 Jan 2016 13:57:33 -0800 Subject: [PATCH 0851/1172] Fix wrong 'dict is too large' on unicode string --- msgpack/_packer.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 0a6513c..6392655 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -174,11 +174,11 @@ cdef class Packer(object): o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) L = len(o) if L > (2**32)-1: - raise ValueError("dict is too large") + raise ValueError("unicode string is too large") rawval = o - ret = msgpack_pack_raw(&self.pk, len(o)) + ret = msgpack_pack_raw(&self.pk, L) if ret == 0: - ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) + ret = msgpack_pack_raw_body(&self.pk, rawval, L) elif PyDict_CheckExact(o): d = o L = len(d) From 1f8240eaf65b28e93621a8e35f1078d4292047f1 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 25 Jan 2016 01:10:50 +0900 Subject: [PATCH 0852/1172] 0.4.7 --- msgpack/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index 2c1c96c..37c172d 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 4, 6) +version = (0, 4, 7) From 3a8bb070f76af211356fef9e0395e1429e81a9b2 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 25 Jan 2016 01:12:56 +0900 Subject: [PATCH 0853/1172] Update ChangeLog --- ChangeLog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 35535b4..f20bb75 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,6 @@ 0.4.7 ===== -:release date: TBD +:release date: 2016-01-25 Bugs fixed ---------- From 005739388d4a611979ae029fab2a1982a366a285 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 25 Jan 2016 01:15:51 +0900 Subject: [PATCH 0854/1172] Drop Python 2.6, 3.2 support --- .travis.yml | 6 ++---- tox.ini | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2ba2caa..eced353 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,10 +11,8 @@ branches: - master env: - - TOXENV=py26-c,py27-c - - TOXENV=py32-c,py33-c,py34-c,py35-c - - TOXENV=py26-pure,py27-pure - - TOXENV=py32-pure,py33-pure,py34-pure,py35-pure + - TOXENV=py27-c,py33-c,py34-c,py35-c + - TOXENV=py27-pure,py33-pure,py34-pure,py35-pure - TOXENV=pypy-pure,pypy3-pure install: diff --git a/tox.ini b/tox.ini index 56ded49..b6e7a7f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py26,py27,py32,py33,py34,py35}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86 +envlist = {py27,py33,py34,py35}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86 [variants:pure] setenv= @@ -36,4 +36,3 @@ commands= python -c 'import sys; print(hex(sys.maxsize))' python -c 'from msgpack import _packer, _unpacker' py.test - From a779b79b47529f84cd71593f284788d939226d66 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 25 Jan 2016 02:18:25 +0900 Subject: [PATCH 0855/1172] Add test for strict_types option --- test/test_stricttype.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 test/test_stricttype.py diff --git a/test/test_stricttype.py b/test/test_stricttype.py new file mode 100644 index 0000000..a20b5eb --- /dev/null +++ b/test/test_stricttype.py @@ -0,0 +1,15 @@ +# coding: utf-8 + +from collections import namedtuple +from msgpack import packb, unpackb + + +def test_namedtuple(): + T = namedtuple('T', "foo bar") + def default(o): + if isinstance(o, T): + return dict(o._asdict()) + raise TypeError('Unsupported type %s' % (type(o),)) + packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default) + unpacked = unpackb(packed, encoding='utf-8') + assert unpacked == {'foo': 1, 'bar': 42} From 31adc5a3c09a5f3506db192e1fb8b7ca4b72d974 Mon Sep 17 00:00:00 2001 From: folz Date: Thu, 12 Nov 2015 11:49:19 +0100 Subject: [PATCH 0856/1172] Support packing memoryview objects --- msgpack/_packer.pyx | 17 +++++++++++++++++ msgpack/fallback.py | 6 ++++-- test/test_memoryview.py | 12 ++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 test/test_memoryview.py diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index c3ef1a4..b19d462 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -10,6 +10,11 @@ from msgpack.exceptions import PackValueError from msgpack import ExtType +cdef extern from "Python.h": + + int PyMemoryView_Check(object obj) + + cdef extern from "pack.h": struct msgpack_packer: char* buf @@ -132,6 +137,7 @@ cdef class Packer(object): cdef size_t L cdef int default_used = 0 cdef bint strict_types = self.strict_types + cdef Py_buffer view if nest_limit < 0: raise PackValueError("recursion limit exceeded.") @@ -231,6 +237,17 @@ cdef class Packer(object): for v in o: ret = self._pack(v, nest_limit-1) if ret != 0: break + elif PyMemoryView_Check(o): + if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: + raise ValueError("could not get buffer for memoryview") + L = view.len + if L > (2**32)-1: + PyBuffer_Release(&view); + raise ValueError("memoryview is too large") + ret = msgpack_pack_bin(&self.pk, L) + if ret == 0: + ret = msgpack_pack_raw_body(&self.pk, view.buf, L) + PyBuffer_Release(&view); elif not default_used and self._default: o = self._default(o) default_used = 1 diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 40c54a8..348e017 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -36,6 +36,8 @@ if hasattr(sys, 'pypy_version_info'): else: self.builder = StringBuilder() def write(self, s): + if isinstance(s, memoryview): + s = s.tobytes() self.builder.append(s) def getvalue(self): return self.builder.build() @@ -682,7 +684,7 @@ class Packer(object): default_used = True continue raise PackValueError("Integer value out of range") - if self._use_bin_type and check(obj, bytes): + if self._use_bin_type and check(obj, (bytes, memoryview)): n = len(obj) if n <= 0xff: self._buffer.write(struct.pack('>BB', 0xc4, n)) @@ -693,7 +695,7 @@ class Packer(object): else: raise PackValueError("Bytes is too large") return self._buffer.write(obj) - if check(obj, (Unicode, bytes)): + if check(obj, (Unicode, bytes, memoryview)): if check(obj, Unicode): if self._encoding is None: raise TypeError( diff --git a/test/test_memoryview.py b/test/test_memoryview.py new file mode 100644 index 0000000..aed5069 --- /dev/null +++ b/test/test_memoryview.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python +# coding: utf-8 + + +from msgpack import packb, unpackb + + +def test_pack_memoryview(): + data = bytearray(range(256)) + view = memoryview(data) + unpacked = unpackb(packb(view)) + assert data == unpacked From 7d2d46effce37f9fbf394fac74d380aaa7c95f02 Mon Sep 17 00:00:00 2001 From: palaviv Date: Fri, 12 Feb 2016 11:00:39 +0200 Subject: [PATCH 0857/1172] msgpack pack and unpack throws only exception that inherit from MsgpackBaseException. cython and fallback throws same exceptions --- msgpack/_packer.pyx | 14 ++++++------ msgpack/exceptions.py | 12 ++++++++-- msgpack/fallback.py | 51 ++++++++++++++++++++++--------------------- test/test_limits.py | 46 +++++++++++++++++++++++--------------- 4 files changed, 71 insertions(+), 52 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index b19d462..8216813 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -6,7 +6,7 @@ from libc.stdlib cimport * from libc.string cimport * from libc.limits cimport * -from msgpack.exceptions import PackValueError +from msgpack.exceptions import PackValueError, PackOverflowError from msgpack import ExtType @@ -166,7 +166,7 @@ cdef class Packer(object): default_used = True continue else: - raise + raise PackOverflowError("Integer value out of range") elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o): longval = o ret = msgpack_pack_long(&self.pk, longval) @@ -180,7 +180,7 @@ cdef class Packer(object): elif PyBytes_CheckExact(o) if strict_types else PyBytes_Check(o): L = len(o) if L > (2**32)-1: - raise ValueError("bytes is too large") + raise PackValueError("bytes is too large") rawval = o ret = msgpack_pack_bin(&self.pk, L) if ret == 0: @@ -191,7 +191,7 @@ cdef class Packer(object): o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) L = len(o) if L > (2**32)-1: - raise ValueError("unicode string is too large") + raise PackValueError("unicode string is too large") rawval = o ret = msgpack_pack_raw(&self.pk, L) if ret == 0: @@ -211,7 +211,7 @@ cdef class Packer(object): elif not strict_types and PyDict_Check(o): L = len(o) if L > (2**32)-1: - raise ValueError("dict is too large") + raise PackValueError("dict is too large") ret = msgpack_pack_map(&self.pk, L) if ret == 0: for k, v in o.items(): @@ -225,13 +225,13 @@ cdef class Packer(object): rawval = o.data L = len(o.data) if L > (2**32)-1: - raise ValueError("EXT data is too large") + raise PackValueError("EXT data is too large") ret = msgpack_pack_ext(&self.pk, longval, L) ret = msgpack_pack_raw_body(&self.pk, rawval, L) elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): L = len(o) if L > (2**32)-1: - raise ValueError("list is too large") + raise PackValueError("list is too large") ret = msgpack_pack_array(&self.pk, L) if ret == 0: for v in o: diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py index f7678f1..e0d5b5f 100644 --- a/msgpack/exceptions.py +++ b/msgpack/exceptions.py @@ -1,4 +1,8 @@ -class UnpackException(Exception): +class MsgpackBaseException(Exception): + pass + + +class UnpackException(MsgpackBaseException): pass @@ -22,8 +26,12 @@ class ExtraData(ValueError): def __str__(self): return "unpack(b) received extra data." -class PackException(Exception): +class PackException(MsgpackBaseException): pass class PackValueError(PackException, ValueError): pass + + +class PackOverflowError(PackValueError, OverflowError): + pass diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 348e017..d8c5d73 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -51,6 +51,7 @@ from msgpack.exceptions import ( OutOfData, UnpackValueError, PackValueError, + PackOverflowError, ExtraData) from msgpack import ExtType @@ -363,17 +364,17 @@ class Unpacker(object): obj = self._fb_read(n, write_bytes) typ = TYPE_RAW if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) elif b & 0b11110000 == 0b10010000: n = b & 0b00001111 typ = TYPE_ARRAY if n > self._max_array_len: - raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) + raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b & 0b11110000 == 0b10000000: n = b & 0b00001111 typ = TYPE_MAP if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) + raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) elif b == 0xc0: obj = None elif b == 0xc2: @@ -384,37 +385,37 @@ class Unpacker(object): typ = TYPE_BIN n = struct.unpack("B", self._fb_read(1, write_bytes))[0] if n > self._max_bin_len: - raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) + raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._fb_read(n, write_bytes) elif b == 0xc5: typ = TYPE_BIN n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] if n > self._max_bin_len: - raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) + raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._fb_read(n, write_bytes) elif b == 0xc6: typ = TYPE_BIN n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] if n > self._max_bin_len: - raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) + raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._fb_read(n, write_bytes) elif b == 0xc7: # ext 8 typ = TYPE_EXT L, n = struct.unpack('Bb', self._fb_read(2, write_bytes)) if L > self._max_ext_len: - raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) + raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._fb_read(L, write_bytes) elif b == 0xc8: # ext 16 typ = TYPE_EXT L, n = struct.unpack('>Hb', self._fb_read(3, write_bytes)) if L > self._max_ext_len: - raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) + raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._fb_read(L, write_bytes) elif b == 0xc9: # ext 32 typ = TYPE_EXT L, n = struct.unpack('>Ib', self._fb_read(5, write_bytes)) if L > self._max_ext_len: - raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) + raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._fb_read(L, write_bytes) elif b == 0xca: obj = struct.unpack(">f", self._fb_read(4, write_bytes))[0] @@ -439,65 +440,65 @@ class Unpacker(object): elif b == 0xd4: # fixext 1 typ = TYPE_EXT if self._max_ext_len < 1: - raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) + raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) n, obj = struct.unpack('b1s', self._fb_read(2, write_bytes)) elif b == 0xd5: # fixext 2 typ = TYPE_EXT if self._max_ext_len < 2: - raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) + raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) n, obj = struct.unpack('b2s', self._fb_read(3, write_bytes)) elif b == 0xd6: # fixext 4 typ = TYPE_EXT if self._max_ext_len < 4: - raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) + raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) n, obj = struct.unpack('b4s', self._fb_read(5, write_bytes)) elif b == 0xd7: # fixext 8 typ = TYPE_EXT if self._max_ext_len < 8: - raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) + raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) n, obj = struct.unpack('b8s', self._fb_read(9, write_bytes)) elif b == 0xd8: # fixext 16 typ = TYPE_EXT if self._max_ext_len < 16: - raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) + raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) n, obj = struct.unpack('b16s', self._fb_read(17, write_bytes)) elif b == 0xd9: typ = TYPE_RAW n = struct.unpack("B", self._fb_read(1, write_bytes))[0] if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._fb_read(n, write_bytes) elif b == 0xda: typ = TYPE_RAW n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._fb_read(n, write_bytes) elif b == 0xdb: typ = TYPE_RAW n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._fb_read(n, write_bytes) elif b == 0xdc: n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] if n > self._max_array_len: - raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) + raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) typ = TYPE_ARRAY elif b == 0xdd: n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] if n > self._max_array_len: - raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) + raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) typ = TYPE_ARRAY elif b == 0xde: n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) + raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP elif b == 0xdf: n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) + raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP else: raise UnpackValueError("Unknown header: 0x%x" % b) @@ -683,7 +684,7 @@ class Packer(object): obj = self._default(obj) default_used = True continue - raise PackValueError("Integer value out of range") + raise PackOverflowError("Integer value out of range") if self._use_bin_type and check(obj, (bytes, memoryview)): n = len(obj) if n <= 0xff: @@ -778,7 +779,7 @@ class Packer(object): def pack_array_header(self, n): if n >= 2**32: - raise ValueError + raise PackValueError self._fb_pack_array_header(n) ret = self._buffer.getvalue() if self._autoreset: @@ -789,7 +790,7 @@ class Packer(object): def pack_map_header(self, n): if n >= 2**32: - raise ValueError + raise PackValueError self._fb_pack_map_header(n) ret = self._buffer.getvalue() if self._autoreset: @@ -807,7 +808,7 @@ class Packer(object): raise TypeError("data must have bytes type") L = len(data) if L > 0xffffffff: - raise ValueError("Too large data") + raise PackValueError("Too large data") if L == 1: self._buffer.write(b'\xd4') elif L == 2: diff --git a/test/test_limits.py b/test/test_limits.py index 3c1cf2a..34acf7c 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -3,36 +3,42 @@ from __future__ import absolute_import, division, print_function, unicode_literals import pytest -from msgpack import packb, unpackb, Packer, Unpacker, ExtType +from msgpack import packb, unpackb, Packer, Unpacker, ExtType, PackException, PackOverflowError, PackValueError +from msgpack import UnpackValueError, UnpackException, MsgpackBaseException -def test_integer(): +@pytest.mark.parametrize("expected_exception", [OverflowError, ValueError, PackOverflowError, + PackException, PackValueError, MsgpackBaseException]) +def test_integer(expected_exception): x = -(2 ** 63) assert unpackb(packb(x)) == x - with pytest.raises((OverflowError, ValueError)): + with pytest.raises(expected_exception): packb(x-1) x = 2 ** 64 - 1 assert unpackb(packb(x)) == x - with pytest.raises((OverflowError, ValueError)): + with pytest.raises(expected_exception): packb(x+1) -def test_array_header(): +@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError, MsgpackBaseException]) +def test_array_header(expected_exception): packer = Packer() packer.pack_array_header(2**32-1) - with pytest.raises((OverflowError, ValueError)): + with pytest.raises(expected_exception): packer.pack_array_header(2**32) -def test_map_header(): +@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError, MsgpackBaseException]) +def test_map_header(expected_exception): packer = Packer() packer.pack_map_header(2**32-1) - with pytest.raises((OverflowError, ValueError)): + with pytest.raises(expected_exception): packer.pack_array_header(2**32) -def test_max_str_len(): +@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException]) +def test_max_str_len(expected_exception): d = 'x' * 3 packed = packb(d) @@ -41,12 +47,13 @@ def test_max_str_len(): assert unpacker.unpack() == d unpacker = Unpacker(max_str_len=2, encoding='utf-8') - with pytest.raises(ValueError): + with pytest.raises(expected_exception): unpacker.feed(packed) unpacker.unpack() -def test_max_bin_len(): +@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException]) +def test_max_bin_len(expected_exception): d = b'x' * 3 packed = packb(d, use_bin_type=True) @@ -55,12 +62,13 @@ def test_max_bin_len(): assert unpacker.unpack() == d unpacker = Unpacker(max_bin_len=2) - with pytest.raises(ValueError): + with pytest.raises(expected_exception): unpacker.feed(packed) unpacker.unpack() -def test_max_array_len(): +@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException]) +def test_max_array_len(expected_exception): d = [1,2,3] packed = packb(d) @@ -69,12 +77,13 @@ def test_max_array_len(): assert unpacker.unpack() == d unpacker = Unpacker(max_array_len=2) - with pytest.raises(ValueError): + with pytest.raises(expected_exception): unpacker.feed(packed) unpacker.unpack() -def test_max_map_len(): +@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException]) +def test_max_map_len(expected_exception): d = {1: 2, 3: 4, 5: 6} packed = packb(d) @@ -83,12 +92,13 @@ def test_max_map_len(): assert unpacker.unpack() == d unpacker = Unpacker(max_map_len=2) - with pytest.raises(ValueError): + with pytest.raises(expected_exception): unpacker.feed(packed) unpacker.unpack() -def test_max_ext_len(): +@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException]) +def test_max_ext_len(expected_exception): d = ExtType(42, b"abc") packed = packb(d) @@ -97,7 +107,7 @@ def test_max_ext_len(): assert unpacker.unpack() == d unpacker = Unpacker(max_ext_len=2) - with pytest.raises(ValueError): + with pytest.raises(expected_exception): unpacker.feed(packed) unpacker.unpack() From d44063119bf11fa5c4b559f9e246df60058bfe31 Mon Sep 17 00:00:00 2001 From: palaviv Date: Fri, 12 Feb 2016 15:36:48 +0200 Subject: [PATCH 0858/1172] changed more ValueErrors to PackValueError --- msgpack/_packer.pyx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 8216813..e923895 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -200,7 +200,7 @@ cdef class Packer(object): d = o L = len(d) if L > (2**32)-1: - raise ValueError("dict is too large") + raise PackValueError("dict is too large") ret = msgpack_pack_map(&self.pk, L) if ret == 0: for k, v in d.iteritems(): @@ -239,11 +239,11 @@ cdef class Packer(object): if ret != 0: break elif PyMemoryView_Check(o): if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: - raise ValueError("could not get buffer for memoryview") + raise PackValueError("could not get buffer for memoryview") L = view.len if L > (2**32)-1: PyBuffer_Release(&view); - raise ValueError("memoryview is too large") + raise PackValueError("memoryview is too large") ret = msgpack_pack_bin(&self.pk, L) if ret == 0: ret = msgpack_pack_raw_body(&self.pk, view.buf, L) @@ -274,7 +274,7 @@ cdef class Packer(object): def pack_array_header(self, size_t size): if size > (2**32-1): - raise ValueError + raise PackValueError cdef int ret = msgpack_pack_array(&self.pk, size) if ret == -1: raise MemoryError @@ -287,7 +287,7 @@ cdef class Packer(object): def pack_map_header(self, size_t size): if size > (2**32-1): - raise ValueError + raise PackValueError cdef int ret = msgpack_pack_map(&self.pk, size) if ret == -1: raise MemoryError From 1183eff688189b0e94ea9e15c5ae13c2f757d745 Mon Sep 17 00:00:00 2001 From: palaviv Date: Fri, 12 Feb 2016 15:37:39 +0200 Subject: [PATCH 0859/1172] reraising ValueError from unpack.h as UnpackValueError --- msgpack/_unpacker.pyx | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 1aefc64..90ebf7d 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -397,24 +397,27 @@ cdef class Unpacker(object): else: raise OutOfData("No more data to unpack.") - ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) - if write_bytes is not None: - write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head)) + try: + ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + if write_bytes is not None: + write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head)) - if ret == 1: - obj = unpack_data(&self.ctx) - unpack_init(&self.ctx) - return obj - elif ret == 0: - if self.file_like is not None: - self.read_from_file() - continue - if iter: - raise StopIteration("No more data to unpack.") + if ret == 1: + obj = unpack_data(&self.ctx) + unpack_init(&self.ctx) + return obj + elif ret == 0: + if self.file_like is not None: + self.read_from_file() + continue + if iter: + raise StopIteration("No more data to unpack.") + else: + raise OutOfData("No more data to unpack.") else: - raise OutOfData("No more data to unpack.") - else: - raise ValueError("Unpack failed: error = %d" % (ret,)) + raise UnpackValueError("Unpack failed: error = %d" % (ret,)) + except ValueError as e: + raise UnpackValueError(e) def read_bytes(self, Py_ssize_t nbytes): """Read a specified number of raw bytes from the stream""" From e15085db0362899520f714e3959c37721c839cef Mon Sep 17 00:00:00 2001 From: palaviv Date: Fri, 12 Feb 2016 15:39:50 +0200 Subject: [PATCH 0860/1172] removed MsgpackBaseException --- msgpack/exceptions.py | 10 ++++------ test/test_limits.py | 18 +++++++++--------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py index e0d5b5f..e982079 100644 --- a/msgpack/exceptions.py +++ b/msgpack/exceptions.py @@ -1,8 +1,4 @@ -class MsgpackBaseException(Exception): - pass - - -class UnpackException(MsgpackBaseException): +class UnpackException(Exception): pass @@ -26,9 +22,11 @@ class ExtraData(ValueError): def __str__(self): return "unpack(b) received extra data." -class PackException(MsgpackBaseException): + +class PackException(Exception): pass + class PackValueError(PackException, ValueError): pass diff --git a/test/test_limits.py b/test/test_limits.py index 34acf7c..e9bc9df 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -4,11 +4,11 @@ from __future__ import absolute_import, division, print_function, unicode_litera import pytest from msgpack import packb, unpackb, Packer, Unpacker, ExtType, PackException, PackOverflowError, PackValueError -from msgpack import UnpackValueError, UnpackException, MsgpackBaseException +from msgpack import UnpackValueError, UnpackException @pytest.mark.parametrize("expected_exception", [OverflowError, ValueError, PackOverflowError, - PackException, PackValueError, MsgpackBaseException]) + PackException, PackValueError]) def test_integer(expected_exception): x = -(2 ** 63) assert unpackb(packb(x)) == x @@ -21,7 +21,7 @@ def test_integer(expected_exception): packb(x+1) -@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError, MsgpackBaseException]) +@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError]) def test_array_header(expected_exception): packer = Packer() packer.pack_array_header(2**32-1) @@ -29,7 +29,7 @@ def test_array_header(expected_exception): packer.pack_array_header(2**32) -@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError, MsgpackBaseException]) +@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError]) def test_map_header(expected_exception): packer = Packer() packer.pack_map_header(2**32-1) @@ -37,7 +37,7 @@ def test_map_header(expected_exception): packer.pack_array_header(2**32) -@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException]) +@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException]) def test_max_str_len(expected_exception): d = 'x' * 3 packed = packb(d) @@ -52,7 +52,7 @@ def test_max_str_len(expected_exception): unpacker.unpack() -@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException]) +@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException]) def test_max_bin_len(expected_exception): d = b'x' * 3 packed = packb(d, use_bin_type=True) @@ -67,7 +67,7 @@ def test_max_bin_len(expected_exception): unpacker.unpack() -@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException]) +@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException]) def test_max_array_len(expected_exception): d = [1,2,3] packed = packb(d) @@ -82,7 +82,7 @@ def test_max_array_len(expected_exception): unpacker.unpack() -@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException]) +@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException]) def test_max_map_len(expected_exception): d = {1: 2, 3: 4, 5: 6} packed = packb(d) @@ -97,7 +97,7 @@ def test_max_map_len(expected_exception): unpacker.unpack() -@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException]) +@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException]) def test_max_ext_len(expected_exception): d = ExtType(42, b"abc") packed = packb(d) From d90008d4f57ec83a15b84cf2db2edc36e9504ac1 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 14 Feb 2016 11:46:28 +0900 Subject: [PATCH 0861/1172] ExtraData should be UnpackValueError --- msgpack/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py index e982079..bb12264 100644 --- a/msgpack/exceptions.py +++ b/msgpack/exceptions.py @@ -14,7 +14,7 @@ class UnpackValueError(UnpackException, ValueError): pass -class ExtraData(ValueError): +class ExtraData(UnpackValueError): def __init__(self, unpacked, extra): self.unpacked = unpacked self.extra = extra From 3dad39811d93a58c5d3de874193290b935da1446 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 14 Feb 2016 11:54:01 +0900 Subject: [PATCH 0862/1172] Deprecate PackExceptions --- msgpack/_unpacker.pyx | 10 +++++----- msgpack/exceptions.py | 12 +++++++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 90ebf7d..0443505 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -11,11 +11,11 @@ from libc.string cimport * from libc.limits cimport * from msgpack.exceptions import ( - BufferFull, - OutOfData, - UnpackValueError, - ExtraData, - ) + BufferFull, + OutOfData, + UnpackValueError, + ExtraData, +) from msgpack import ExtType diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py index bb12264..73010b7 100644 --- a/msgpack/exceptions.py +++ b/msgpack/exceptions.py @@ -24,12 +24,18 @@ class ExtraData(UnpackValueError): class PackException(Exception): - pass + """Deprecated. Use Exception instead to catch all exception during packing.""" class PackValueError(PackException, ValueError): - pass + """PackValueError is raised when type of input data is supported but it's value is unsupported. + + Deprecated. Use ValueError instead. + """ class PackOverflowError(PackValueError, OverflowError): - pass + """PackOverflowError is raised when integer value is out of range of msgpack support [-2**31, 2**32). + + Deprecated. Use ValueError instead. + """ From 6e364762394fdb06d0453411a5f020ee594c06b0 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 14 Feb 2016 11:58:56 +0900 Subject: [PATCH 0863/1172] remove too much parameterized tests --- test/test_limits.py | 49 +++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/test/test_limits.py b/test/test_limits.py index e9bc9df..197ef46 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -3,42 +3,39 @@ from __future__ import absolute_import, division, print_function, unicode_literals import pytest -from msgpack import packb, unpackb, Packer, Unpacker, ExtType, PackException, PackOverflowError, PackValueError -from msgpack import UnpackValueError, UnpackException +from msgpack import ( + packb, unpackb, Packer, Unpacker, ExtType, + PackOverflowError, PackValueError, UnpackValueError, +) -@pytest.mark.parametrize("expected_exception", [OverflowError, ValueError, PackOverflowError, - PackException, PackValueError]) -def test_integer(expected_exception): +def test_integer(): x = -(2 ** 63) assert unpackb(packb(x)) == x - with pytest.raises(expected_exception): + with pytest.raises(PackOverflowError): packb(x-1) x = 2 ** 64 - 1 assert unpackb(packb(x)) == x - with pytest.raises(expected_exception): + with pytest.raises(PackOverflowError): packb(x+1) -@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError]) -def test_array_header(expected_exception): +def test_array_header(): packer = Packer() packer.pack_array_header(2**32-1) - with pytest.raises(expected_exception): + with pytest.raises(PackValueError): packer.pack_array_header(2**32) -@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError]) -def test_map_header(expected_exception): +def test_map_header(): packer = Packer() packer.pack_map_header(2**32-1) - with pytest.raises(expected_exception): + with pytest.raises(PackValueError): packer.pack_array_header(2**32) -@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException]) -def test_max_str_len(expected_exception): +def test_max_str_len(): d = 'x' * 3 packed = packb(d) @@ -47,13 +44,12 @@ def test_max_str_len(expected_exception): assert unpacker.unpack() == d unpacker = Unpacker(max_str_len=2, encoding='utf-8') - with pytest.raises(expected_exception): + with pytest.raises(UnpackValueError): unpacker.feed(packed) unpacker.unpack() -@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException]) -def test_max_bin_len(expected_exception): +def test_max_bin_len(): d = b'x' * 3 packed = packb(d, use_bin_type=True) @@ -62,13 +58,12 @@ def test_max_bin_len(expected_exception): assert unpacker.unpack() == d unpacker = Unpacker(max_bin_len=2) - with pytest.raises(expected_exception): + with pytest.raises(UnpackValueError): unpacker.feed(packed) unpacker.unpack() -@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException]) -def test_max_array_len(expected_exception): +def test_max_array_len(): d = [1,2,3] packed = packb(d) @@ -77,13 +72,12 @@ def test_max_array_len(expected_exception): assert unpacker.unpack() == d unpacker = Unpacker(max_array_len=2) - with pytest.raises(expected_exception): + with pytest.raises(UnpackValueError): unpacker.feed(packed) unpacker.unpack() -@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException]) -def test_max_map_len(expected_exception): +def test_max_map_len(): d = {1: 2, 3: 4, 5: 6} packed = packb(d) @@ -92,13 +86,12 @@ def test_max_map_len(expected_exception): assert unpacker.unpack() == d unpacker = Unpacker(max_map_len=2) - with pytest.raises(expected_exception): + with pytest.raises(UnpackValueError): unpacker.feed(packed) unpacker.unpack() -@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException]) -def test_max_ext_len(expected_exception): +def test_max_ext_len(): d = ExtType(42, b"abc") packed = packb(d) @@ -107,7 +100,7 @@ def test_max_ext_len(expected_exception): assert unpacker.unpack() == d unpacker = Unpacker(max_ext_len=2) - with pytest.raises(expected_exception): + with pytest.raises(UnpackValueError): unpacker.feed(packed) unpacker.unpack() From b2a8ce6cbdbef80d1a89d02fa483f56862cf1efa Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 14 Feb 2016 14:32:11 +0900 Subject: [PATCH 0864/1172] Deprecate more useless exceptions --- msgpack/exceptions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py index 73010b7..9766881 100644 --- a/msgpack/exceptions.py +++ b/msgpack/exceptions.py @@ -1,5 +1,5 @@ class UnpackException(Exception): - pass + """Deprecated. Use Exception instead to catch all exception during unpacking.""" class BufferFull(UnpackException): @@ -11,7 +11,7 @@ class OutOfData(UnpackException): class UnpackValueError(UnpackException, ValueError): - pass + """Deprecated. Use ValueError instead.""" class ExtraData(UnpackValueError): From 2192310bc4a7af32a628191ededd1feeec624845 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 16 Apr 2016 02:03:18 +0900 Subject: [PATCH 0865/1172] Use manylinux1 wheel for Cython (#179) * Use manylinux1 wheel for Cython * Use newer pip --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index eced353..e7e9b63 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,8 @@ env: - TOXENV=pypy-pure,pypy3-pure install: - - pip install tox - - pip install cython --install-option=--cython-with-refnanny --install-option=--no-cython-compile + - pip install -U pip + - pip install tox cython - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx script: tox From 40ee322440a018c9e09634aa2c190d1747d7f0bd Mon Sep 17 00:00:00 2001 From: Timothy Cyrus Date: Fri, 29 Apr 2016 11:18:27 -0400 Subject: [PATCH 0866/1172] Update README.rst (#184) Change PNG Badges to SVG --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d32ec1d..8136ccb 100644 --- a/README.rst +++ b/README.rst @@ -6,7 +6,7 @@ MessagePack for Python :version: 0.4.6 :date: 2015-03-13 -.. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png +.. image:: https://secure.travis-ci.org/msgpack/msgpack-python.svg :target: https://travis-ci.org/#!/msgpack/msgpack-python From 6b113a6fb37ffb969e92429b06aab9ea9b8eeb4a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 30 Apr 2016 17:07:14 +0900 Subject: [PATCH 0867/1172] Use Python's memory API (#185) --- msgpack/_packer.pyx | 7 +++---- msgpack/_unpacker.pyx | 23 ++++++++++++++++++----- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index e923895..d491cc1 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -88,7 +88,7 @@ cdef class Packer(object): def __cinit__(self): cdef int buf_size = 1024*1024 - self.pk.buf = malloc(buf_size); + self.pk.buf = PyMem_Malloc(buf_size) if self.pk.buf == NULL: raise MemoryError("Unable to allocate internal buffer.") self.pk.buf_size = buf_size @@ -97,8 +97,6 @@ cdef class Packer(object): def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, bint autoreset=1, bint use_bin_type=0, bint strict_types=0): - """ - """ self.use_float = use_single_float self.strict_types = strict_types self.autoreset = autoreset @@ -123,7 +121,8 @@ cdef class Packer(object): self.unicode_errors = PyBytes_AsString(self._berrors) def __dealloc__(self): - free(self.pk.buf); + PyMem_Free(self.pk.buf) + self.pk.buf = NULL cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: cdef long long llval diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 0443505..23f6478 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -1,7 +1,20 @@ # coding: utf-8 #cython: embedsignature=True -from cpython cimport * +from cpython.bytes cimport ( + PyBytes_AsString, + PyBytes_FromStringAndSize, + PyBytes_Size, +) +from cpython.buffer cimport ( + Py_buffer, + PyBuffer_Release, + PyObject_GetBuffer, + PyBUF_SIMPLE, +) +from cpython.mem cimport PyMem_Malloc, PyMem_Free +from cpython.object cimport PyCallable_Check + cdef extern from "Python.h": ctypedef struct PyObject cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1 @@ -256,7 +269,7 @@ cdef class Unpacker(object): self.buf = NULL def __dealloc__(self): - free(self.buf) + PyMem_Free(self.buf) self.buf = NULL def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1, @@ -289,7 +302,7 @@ cdef class Unpacker(object): read_size = min(max_buffer_size, 1024**2) self.max_buffer_size = max_buffer_size self.read_size = read_size - self.buf = malloc(read_size) + self.buf = PyMem_Malloc(read_size) if self.buf == NULL: raise MemoryError("Unable to allocate internal buffer.") self.buf_size = read_size @@ -352,13 +365,13 @@ cdef class Unpacker(object): if new_size > self.max_buffer_size: raise BufferFull new_size = min(new_size*2, self.max_buffer_size) - new_buf = malloc(new_size) + new_buf = PyMem_Malloc(new_size) if new_buf == NULL: # self.buf still holds old buffer and will be freed during # obj destruction raise MemoryError("Unable to enlarge internal buffer.") memcpy(new_buf, buf + head, tail - head) - free(buf) + PyMem_Free(buf) buf = new_buf buf_size = new_size From ceb9635a3f4d7f3dd4874b98773ca6f7db9296a7 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 3 May 2016 11:58:28 +0900 Subject: [PATCH 0868/1172] Use AppVeyor to build windows wheel (#188) * Add AppVeyor support to build windows wheel * Fix test_limits on 32bit environments * Ignore Python35-x64 test fail for now Should be fixed in next version. --- appveyor.yml | 57 +++++++++++++++++++++++++++++++++++++++++++++ build.cmd | 21 +++++++++++++++++ msgpack/_packer.pyx | 4 ++-- 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 appveyor.yml create mode 100644 build.cmd diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..02b4461 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,57 @@ +environment: + + matrix: + + # For Python versions available on Appveyor, see + # http://www.appveyor.com/docs/installed-software#python + # The list here is complete (excluding Python 2.6, which + # isn't covered by this document) at the time of writing. + + - PYTHON: "C:\\Python27" + - PYTHON: "C:\\Python34" + - PYTHON: "C:\\Python35" + - PYTHON: "C:\\Python27-x64" + - PYTHON: "C:\\Python34-x64" + DISTUTILS_USE_SDK: "1" + + # Python35-x64 test fails with MemoryError + # TODO: investigate it + #- PYTHON: "C:\\Python35-x64" + +install: + # We need wheel installed to build wheels + - "%PYTHON%\\python.exe -m pip install -U pip wheel pytest cython" + +build: off + +test_script: + # Put your test command here. + # If you don't need to build C extensions on 64-bit Python 3.3 or 3.4, + # you can remove "build.cmd" from the front of the command, as it's + # only needed to support those cases. + # Note that you must use the environment variable %PYTHON% to refer to + # the interpreter you're using - Appveyor does not do anything special + # to put the Python evrsion you want to use on PATH. + - "build.cmd %PYTHON%\\python.exe setup.py build_ext -i" + - "build.cmd %PYTHON%\\python.exe setup.py install" + - "%PYTHON%\\python.exe -c \"import sys; print(hex(sys.maxsize))\"" + - "%PYTHON%\\python.exe -c \"from msgpack import _packer, _unpacker\"" + - "%PYTHON%\\Scripts\\py.test test" + - "build.cmd %PYTHON%\\python.exe setup.py bdist_wheel" + +after_test: + # This step builds your wheels. + # Again, you only need build.cmd if you're building C extensions for + # 64-bit Python 3.3/3.4. And you need to use %PYTHON% to get the correct + # interpreter + +artifacts: + # bdist_wheel puts your built wheel in the dist directory + - path: dist\* + +#on_success: +# You can use this step to upload your artifacts to a public website. +# See Appveyor's documentation for more details. Or you can simply +# access your wheels from the Appveyor "artifacts" tab for your build. + +# vim: set shiftwidth=2 diff --git a/build.cmd b/build.cmd new file mode 100644 index 0000000..243dc9a --- /dev/null +++ b/build.cmd @@ -0,0 +1,21 @@ +@echo off +:: To build extensions for 64 bit Python 3, we need to configure environment +:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 4 +:: +:: More details at: +:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows + +IF "%DISTUTILS_USE_SDK%"=="1" ( + ECHO Configuring environment to build with MSVC on a 64bit architecture + ECHO Using Windows SDK 7.1 + "C:\Program Files\Microsoft SDKs\Windows\v7.1\Setup\WindowsSdkVer.exe" -q -version:v7.1 + CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 /release + SET MSSdk=1 + REM Need the following to allow tox to see the SDK compiler + SET TOX_TESTENV_PASSENV=DISTUTILS_USE_SDK MSSdk INCLUDE LIB +) ELSE ( + ECHO Using default MSVC build environment +) + +CALL %* diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index d491cc1..b1a912b 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -271,7 +271,7 @@ cdef class Packer(object): msgpack_pack_ext(&self.pk, typecode, len(data)) msgpack_pack_raw_body(&self.pk, data, len(data)) - def pack_array_header(self, size_t size): + def pack_array_header(self, long long size): if size > (2**32-1): raise PackValueError cdef int ret = msgpack_pack_array(&self.pk, size) @@ -284,7 +284,7 @@ cdef class Packer(object): self.pk.length = 0 return buf - def pack_map_header(self, size_t size): + def pack_map_header(self, long long size): if size > (2**32-1): raise PackValueError cdef int ret = msgpack_pack_map(&self.pk, size) From 0ec2e3534f9b7751be484bd2f1344e24c49bb24f Mon Sep 17 00:00:00 2001 From: folz Date: Thu, 28 Apr 2016 15:08:28 +0200 Subject: [PATCH 0869/1172] fix problems associated with packing memoryviews fix wrong length when packing multibyte memoryviews in fallback add tests for memoryviews of different types and sizes and check contents of packed data --- msgpack/_packer.pyx | 23 +++---- msgpack/fallback.py | 26 +++++++- test/test_memoryview.py | 133 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 164 insertions(+), 18 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index b1a912b..e07b194 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -39,6 +39,7 @@ cdef extern from "pack.h": int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l) cdef int DEFAULT_RECURSE_LIMIT=511 +cdef size_t ITEM_LIMIT = (2**32)-1 cdef class Packer(object): @@ -178,7 +179,7 @@ cdef class Packer(object): ret = msgpack_pack_double(&self.pk, dval) elif PyBytes_CheckExact(o) if strict_types else PyBytes_Check(o): L = len(o) - if L > (2**32)-1: + if L > ITEM_LIMIT: raise PackValueError("bytes is too large") rawval = o ret = msgpack_pack_bin(&self.pk, L) @@ -189,7 +190,7 @@ cdef class Packer(object): raise TypeError("Can't encode unicode string: no encoding is specified") o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) L = len(o) - if L > (2**32)-1: + if L > ITEM_LIMIT: raise PackValueError("unicode string is too large") rawval = o ret = msgpack_pack_raw(&self.pk, L) @@ -198,7 +199,7 @@ cdef class Packer(object): elif PyDict_CheckExact(o): d = o L = len(d) - if L > (2**32)-1: + if L > ITEM_LIMIT: raise PackValueError("dict is too large") ret = msgpack_pack_map(&self.pk, L) if ret == 0: @@ -209,7 +210,7 @@ cdef class Packer(object): if ret != 0: break elif not strict_types and PyDict_Check(o): L = len(o) - if L > (2**32)-1: + if L > ITEM_LIMIT: raise PackValueError("dict is too large") ret = msgpack_pack_map(&self.pk, L) if ret == 0: @@ -223,13 +224,13 @@ cdef class Packer(object): longval = o.code rawval = o.data L = len(o.data) - if L > (2**32)-1: + if L > ITEM_LIMIT: raise PackValueError("EXT data is too large") ret = msgpack_pack_ext(&self.pk, longval, L) ret = msgpack_pack_raw_body(&self.pk, rawval, L) elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): L = len(o) - if L > (2**32)-1: + if L > ITEM_LIMIT: raise PackValueError("list is too large") ret = msgpack_pack_array(&self.pk, L) if ret == 0: @@ -240,7 +241,7 @@ cdef class Packer(object): if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: raise PackValueError("could not get buffer for memoryview") L = view.len - if L > (2**32)-1: + if L > ITEM_LIMIT: PyBuffer_Release(&view); raise PackValueError("memoryview is too large") ret = msgpack_pack_bin(&self.pk, L) @@ -271,8 +272,8 @@ cdef class Packer(object): msgpack_pack_ext(&self.pk, typecode, len(data)) msgpack_pack_raw_body(&self.pk, data, len(data)) - def pack_array_header(self, long long size): - if size > (2**32-1): + def pack_array_header(self, size_t size): + if size > ITEM_LIMIT: raise PackValueError cdef int ret = msgpack_pack_array(&self.pk, size) if ret == -1: @@ -284,8 +285,8 @@ cdef class Packer(object): self.pk.length = 0 return buf - def pack_map_header(self, long long size): - if size > (2**32-1): + def pack_map_header(self, size_t size): + if size > ITEM_LIMIT: raise PackValueError cdef int ret = msgpack_pack_map(&self.pk, size) if ret == -1: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index d8c5d73..db47d5c 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -685,7 +685,7 @@ class Packer(object): default_used = True continue raise PackOverflowError("Integer value out of range") - if self._use_bin_type and check(obj, (bytes, memoryview)): + if self._use_bin_type and check(obj, bytes): n = len(obj) if n <= 0xff: self._buffer.write(struct.pack('>BB', 0xc4, n)) @@ -696,7 +696,7 @@ class Packer(object): else: raise PackValueError("Bytes is too large") return self._buffer.write(obj) - if check(obj, (Unicode, bytes, memoryview)): + if check(obj, (Unicode, bytes)): if check(obj, Unicode): if self._encoding is None: raise TypeError( @@ -715,6 +715,28 @@ class Packer(object): else: raise PackValueError("String is too large") return self._buffer.write(obj) + if check(obj, memoryview): + n = len(obj) * obj.itemsize + if self._use_bin_type: + if n <= 0xff: + self._buffer.write(struct.pack('>BB', 0xc4, n)) + elif n <= 0xffff: + self._buffer.write(struct.pack(">BH", 0xc5, n)) + elif n <= 0xffffffff: + self._buffer.write(struct.pack(">BI", 0xc6, n)) + else: + raise PackValueError("memoryview is too large") + return self._buffer.write(obj) + else: + if n <= 0x1f: + self._buffer.write(struct.pack('B', 0xa0 + n)) + elif n <= 0xffff: + self._buffer.write(struct.pack(">BH", 0xda, n)) + elif n <= 0xffffffff: + self._buffer.write(struct.pack(">BI", 0xdb, n)) + else: + raise PackValueError("memoryview is too large") + return self._buffer.write(obj) if check(obj, float): if self._use_float: return self._buffer.write(struct.pack(">Bf", 0xca, obj)) diff --git a/test/test_memoryview.py b/test/test_memoryview.py index aed5069..f555c5b 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -2,11 +2,134 @@ # coding: utf-8 +from array import array from msgpack import packb, unpackb +import sys -def test_pack_memoryview(): - data = bytearray(range(256)) - view = memoryview(data) - unpacked = unpackb(packb(view)) - assert data == unpacked +# For Python < 3: +# - array type only supports old buffer interface +# - array.frombytes is not available, must use deprecated array.fromstring +if sys.version_info[0] < 3: + def __memoryview(obj): + return memoryview(buffer(obj)) + + def __make_array(f, data): + a = array(f) + a.fromstring(data) + return a + + def __get_data(a): + return a.tostring() +else: + __memoryview = memoryview + + def __make_array(f, data): + a = array(f) + a.frombytes(data) + return a + + def __get_data(a): + return a.tobytes() + + +def __run_test(format, nbytes, expected_header, expected_prefix, use_bin_type): + # create a new array + original_array = array(format) + original_array.fromlist([255] * (nbytes // original_array.itemsize)) + original_data = __get_data(original_array) + view = __memoryview(original_array) + + # pack, unpack, and reconstruct array + packed = packb(view, use_bin_type=use_bin_type) + unpacked = unpackb(packed) + reconstructed_array = __make_array(format, unpacked) + + # check that we got the right amount of data + assert len(original_data) == nbytes + # check packed header + assert packed[:1] == expected_header + # check packed length prefix, if any + assert packed[1:1+len(expected_prefix)] == expected_prefix + # check packed data + assert packed[1+len(expected_prefix):] == original_data + # check array unpacked correctly + assert original_array == reconstructed_array + + +# ----------- +# test fixstr +# ----------- + + +def test_memoryview_byte_fixstr(): + __run_test('B', 31, b'\xbf', b'', False) + + +def test_memoryview_float_fixstr(): + __run_test('f', 28, b'\xbc', b'', False) + + +# ---------- +# test str16 +# ---------- + + +def test_memoryview_byte_str16(): + __run_test('B', 2**8, b'\xda', b'\x01\x00', False) + + +def test_memoryview_float_str16(): + __run_test('f', 2**8, b'\xda', b'\x01\x00', False) + + +# ---------- +# test str32 +# ---------- + + +def test_memoryview_byte_str32(): + __run_test('B', 2**16, b'\xdb', b'\x00\x01\x00\x00', False) + + +def test_memoryview_float_str32(): + __run_test('f', 2**16, b'\xdb', b'\x00\x01\x00\x00', False) + + +# --------- +# test bin8 +# --------- + + +def test_memoryview_byte_bin8(): + __run_test('B', 1, b'\xc4', b'\x01', True) + + +def test_memoryview_float_bin8(): + __run_test('f', 4, b'\xc4', b'\x04', True) + + +# ---------- +# test bin16 +# ---------- + + +def test_memoryview_byte_bin16(): + __run_test('B', 2**8, b'\xc5', b'\x01\x00', True) + + +def test_memoryview_float_bin16(): + __run_test('f', 2**8, b'\xc5', b'\x01\x00', True) + + +# ---------- +# test bin32 +# ---------- + + +def test_memoryview_byte_bin32(): + __run_test('B', 2**16, b'\xc6', b'\x00\x01\x00\x00', True) + + +def test_memoryview_float_bin32(): + __run_test('f', 2**16, b'\xc6', b'\x00\x01\x00\x00', True) From 0b55989f0b045f1a77d4230bea3b6da70eb3d840 Mon Sep 17 00:00:00 2001 From: folz Date: Wed, 4 May 2016 10:04:09 +0200 Subject: [PATCH 0870/1172] more descriptive test names --- test/test_memoryview.py | 72 ++++++++++++----------------------------- 1 file changed, 21 insertions(+), 51 deletions(-) diff --git a/test/test_memoryview.py b/test/test_memoryview.py index f555c5b..2768867 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -11,25 +11,25 @@ import sys # - array type only supports old buffer interface # - array.frombytes is not available, must use deprecated array.fromstring if sys.version_info[0] < 3: - def __memoryview(obj): + def make_memoryview(obj): return memoryview(buffer(obj)) - def __make_array(f, data): + def make_array(f, data): a = array(f) a.fromstring(data) return a - def __get_data(a): + def get_data(a): return a.tostring() else: - __memoryview = memoryview + make_memoryview = memoryview - def __make_array(f, data): + def make_array(f, data): a = array(f) a.frombytes(data) return a - def __get_data(a): + def get_data(a): return a.tobytes() @@ -37,13 +37,13 @@ def __run_test(format, nbytes, expected_header, expected_prefix, use_bin_type): # create a new array original_array = array(format) original_array.fromlist([255] * (nbytes // original_array.itemsize)) - original_data = __get_data(original_array) - view = __memoryview(original_array) + original_data = get_data(original_array) + view = make_memoryview(original_array) # pack, unpack, and reconstruct array packed = packb(view, use_bin_type=use_bin_type) unpacked = unpackb(packed) - reconstructed_array = __make_array(format, unpacked) + reconstructed_array = make_array(format, unpacked) # check that we got the right amount of data assert len(original_data) == nbytes @@ -57,79 +57,49 @@ def __run_test(format, nbytes, expected_header, expected_prefix, use_bin_type): assert original_array == reconstructed_array -# ----------- -# test fixstr -# ----------- - - -def test_memoryview_byte_fixstr(): +def test_fixstr_from_byte(): __run_test('B', 31, b'\xbf', b'', False) -def test_memoryview_float_fixstr(): +def test_fixstr_from_float(): __run_test('f', 28, b'\xbc', b'', False) -# ---------- -# test str16 -# ---------- - - -def test_memoryview_byte_str16(): +def test_str16_from_byte(): __run_test('B', 2**8, b'\xda', b'\x01\x00', False) -def test_memoryview_float_str16(): +def test_str16_from_float(): __run_test('f', 2**8, b'\xda', b'\x01\x00', False) -# ---------- -# test str32 -# ---------- - - -def test_memoryview_byte_str32(): +def test_str32_from_byte(): __run_test('B', 2**16, b'\xdb', b'\x00\x01\x00\x00', False) -def test_memoryview_float_str32(): +def test_str32_from_float(): __run_test('f', 2**16, b'\xdb', b'\x00\x01\x00\x00', False) -# --------- -# test bin8 -# --------- - - -def test_memoryview_byte_bin8(): +def test_bin8_from_byte(): __run_test('B', 1, b'\xc4', b'\x01', True) -def test_memoryview_float_bin8(): +def test_bin8_from_float(): __run_test('f', 4, b'\xc4', b'\x04', True) -# ---------- -# test bin16 -# ---------- - - -def test_memoryview_byte_bin16(): +def test_bin16_from_byte(): __run_test('B', 2**8, b'\xc5', b'\x01\x00', True) -def test_memoryview_float_bin16(): +def test_bin16_from_float(): __run_test('f', 2**8, b'\xc5', b'\x01\x00', True) -# ---------- -# test bin32 -# ---------- - - -def test_memoryview_byte_bin32(): +def test_bin32_from_byte(): __run_test('B', 2**16, b'\xc6', b'\x00\x01\x00\x00', True) -def test_memoryview_float_bin32(): +def test_bin32_from_float(): __run_test('f', 2**16, b'\xc6', b'\x00\x01\x00\x00', True) From 5860af953ae1c3f459ddc589cd815ec195db46a9 Mon Sep 17 00:00:00 2001 From: folz Date: Wed, 4 May 2016 11:01:27 +0200 Subject: [PATCH 0871/1172] refactor header packing for str and bin types --- msgpack/fallback.py | 83 +++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 44 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index db47d5c..abed3d9 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -685,58 +685,29 @@ class Packer(object): default_used = True continue raise PackOverflowError("Integer value out of range") - if self._use_bin_type and check(obj, bytes): + if check(obj, bytes): n = len(obj) - if n <= 0xff: - self._buffer.write(struct.pack('>BB', 0xc4, n)) - elif n <= 0xffff: - self._buffer.write(struct.pack(">BH", 0xc5, n)) - elif n <= 0xffffffff: - self._buffer.write(struct.pack(">BI", 0xc6, n)) - else: + if n >= 2**32: raise PackValueError("Bytes is too large") + self._fb_pack_bin_header(n) return self._buffer.write(obj) - if check(obj, (Unicode, bytes)): - if check(obj, Unicode): - if self._encoding is None: - raise TypeError( - "Can't encode unicode string: " - "no encoding is specified") - obj = obj.encode(self._encoding, self._unicode_errors) + if check(obj, Unicode): + if self._encoding is None: + raise TypeError( + "Can't encode unicode string: " + "no encoding is specified") + obj = obj.encode(self._encoding, self._unicode_errors) n = len(obj) - if n <= 0x1f: - self._buffer.write(struct.pack('B', 0xa0 + n)) - elif self._use_bin_type and n <= 0xff: - self._buffer.write(struct.pack('>BB', 0xd9, n)) - elif n <= 0xffff: - self._buffer.write(struct.pack(">BH", 0xda, n)) - elif n <= 0xffffffff: - self._buffer.write(struct.pack(">BI", 0xdb, n)) - else: + if n >= 2**32: raise PackValueError("String is too large") + self._fb_pack_raw_header(n) return self._buffer.write(obj) if check(obj, memoryview): n = len(obj) * obj.itemsize - if self._use_bin_type: - if n <= 0xff: - self._buffer.write(struct.pack('>BB', 0xc4, n)) - elif n <= 0xffff: - self._buffer.write(struct.pack(">BH", 0xc5, n)) - elif n <= 0xffffffff: - self._buffer.write(struct.pack(">BI", 0xc6, n)) - else: - raise PackValueError("memoryview is too large") - return self._buffer.write(obj) - else: - if n <= 0x1f: - self._buffer.write(struct.pack('B', 0xa0 + n)) - elif n <= 0xffff: - self._buffer.write(struct.pack(">BH", 0xda, n)) - elif n <= 0xffffffff: - self._buffer.write(struct.pack(">BI", 0xdb, n)) - else: - raise PackValueError("memoryview is too large") - return self._buffer.write(obj) + if n >= 2**32: + raise PackValueError("Memoryview is too large") + self._fb_pack_bin_header(n) + return self._buffer.write(obj) if check(obj, float): if self._use_float: return self._buffer.write(struct.pack(">Bf", 0xca, obj)) @@ -874,6 +845,30 @@ class Packer(object): self._pack(k, nest_limit - 1) self._pack(v, nest_limit - 1) + def _fb_pack_raw_header(self, n): + if n <= 0x1f: + self._buffer.write(struct.pack('B', 0xa0 + n)) + elif self._use_bin_type and n <= 0xff: + self._buffer.write(struct.pack('>BB', 0xd9, n)) + elif n <= 0xffff: + self._buffer.write(struct.pack(">BH", 0xda, n)) + elif n <= 0xffffffff: + self._buffer.write(struct.pack(">BI", 0xdb, n)) + else: + raise PackValueError('Raw is too large') + + def _fb_pack_bin_header(self, n): + if not self._use_bin_type: + return self._fb_pack_raw_header(n) + elif n <= 0xff: + return self._buffer.write(struct.pack('>BB', 0xc4, n)) + elif n <= 0xffff: + return self._buffer.write(struct.pack(">BH", 0xc5, n)) + elif n <= 0xffffffff: + return self._buffer.write(struct.pack(">BI", 0xc6, n)) + else: + raise PackValueError('Bin is too large') + def bytes(self): return self._buffer.getvalue() From a91d5c538ea5bbee0f00ff180a8e72d27df6cfc1 Mon Sep 17 00:00:00 2001 From: folz Date: Wed, 4 May 2016 12:03:37 +0200 Subject: [PATCH 0872/1172] add lower bound tests for memoryviews --- test/test_memoryview.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/test_memoryview.py b/test/test_memoryview.py index 2768867..7ce6bfc 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -58,19 +58,23 @@ def __run_test(format, nbytes, expected_header, expected_prefix, use_bin_type): def test_fixstr_from_byte(): + __run_test('B', 1, b'\xa1', b'', False) __run_test('B', 31, b'\xbf', b'', False) def test_fixstr_from_float(): + __run_test('f', 4, b'\xa4', b'', False) __run_test('f', 28, b'\xbc', b'', False) def test_str16_from_byte(): __run_test('B', 2**8, b'\xda', b'\x01\x00', False) + __run_test('B', 2**16-1, b'\xda', b'\xff\xff', False) def test_str16_from_float(): __run_test('f', 2**8, b'\xda', b'\x01\x00', False) + __run_test('f', 2**16-4, b'\xda', b'\xff\xfc', False) def test_str32_from_byte(): @@ -83,18 +87,22 @@ def test_str32_from_float(): def test_bin8_from_byte(): __run_test('B', 1, b'\xc4', b'\x01', True) + __run_test('B', 2**8-1, b'\xc4', b'\xff', True) def test_bin8_from_float(): __run_test('f', 4, b'\xc4', b'\x04', True) + __run_test('f', 2**8-4, b'\xc4', b'\xfc', True) def test_bin16_from_byte(): __run_test('B', 2**8, b'\xc5', b'\x01\x00', True) + __run_test('B', 2**16-1, b'\xc5', b'\xff\xff', True) def test_bin16_from_float(): __run_test('f', 2**8, b'\xc5', b'\x01\x00', True) + __run_test('f', 2**16-4, b'\xc5', b'\xff\xfc', True) def test_bin32_from_byte(): From 53f47ef55d8d93e276ecf9041a9a8b43fc041aef Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 5 May 2016 00:49:48 +0900 Subject: [PATCH 0873/1172] Remove double underscore prefix --- test/test_memoryview.py | 43 ++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/test/test_memoryview.py b/test/test_memoryview.py index 7ce6bfc..f6d74ed 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # coding: utf-8 - from array import array from msgpack import packb, unpackb import sys @@ -33,7 +32,7 @@ else: return a.tobytes() -def __run_test(format, nbytes, expected_header, expected_prefix, use_bin_type): +def _runtest(format, nbytes, expected_header, expected_prefix, use_bin_type): # create a new array original_array = array(format) original_array.fromlist([255] * (nbytes // original_array.itemsize)) @@ -58,56 +57,56 @@ def __run_test(format, nbytes, expected_header, expected_prefix, use_bin_type): def test_fixstr_from_byte(): - __run_test('B', 1, b'\xa1', b'', False) - __run_test('B', 31, b'\xbf', b'', False) + _runtest('B', 1, b'\xa1', b'', False) + _runtest('B', 31, b'\xbf', b'', False) def test_fixstr_from_float(): - __run_test('f', 4, b'\xa4', b'', False) - __run_test('f', 28, b'\xbc', b'', False) + _runtest('f', 4, b'\xa4', b'', False) + _runtest('f', 28, b'\xbc', b'', False) def test_str16_from_byte(): - __run_test('B', 2**8, b'\xda', b'\x01\x00', False) - __run_test('B', 2**16-1, b'\xda', b'\xff\xff', False) + _runtest('B', 2**8, b'\xda', b'\x01\x00', False) + _runtest('B', 2**16-1, b'\xda', b'\xff\xff', False) def test_str16_from_float(): - __run_test('f', 2**8, b'\xda', b'\x01\x00', False) - __run_test('f', 2**16-4, b'\xda', b'\xff\xfc', False) + _runtest('f', 2**8, b'\xda', b'\x01\x00', False) + _runtest('f', 2**16-4, b'\xda', b'\xff\xfc', False) def test_str32_from_byte(): - __run_test('B', 2**16, b'\xdb', b'\x00\x01\x00\x00', False) + _runtest('B', 2**16, b'\xdb', b'\x00\x01\x00\x00', False) def test_str32_from_float(): - __run_test('f', 2**16, b'\xdb', b'\x00\x01\x00\x00', False) + _runtest('f', 2**16, b'\xdb', b'\x00\x01\x00\x00', False) def test_bin8_from_byte(): - __run_test('B', 1, b'\xc4', b'\x01', True) - __run_test('B', 2**8-1, b'\xc4', b'\xff', True) + _runtest('B', 1, b'\xc4', b'\x01', True) + _runtest('B', 2**8-1, b'\xc4', b'\xff', True) def test_bin8_from_float(): - __run_test('f', 4, b'\xc4', b'\x04', True) - __run_test('f', 2**8-4, b'\xc4', b'\xfc', True) + _runtest('f', 4, b'\xc4', b'\x04', True) + _runtest('f', 2**8-4, b'\xc4', b'\xfc', True) def test_bin16_from_byte(): - __run_test('B', 2**8, b'\xc5', b'\x01\x00', True) - __run_test('B', 2**16-1, b'\xc5', b'\xff\xff', True) + _runtest('B', 2**8, b'\xc5', b'\x01\x00', True) + _runtest('B', 2**16-1, b'\xc5', b'\xff\xff', True) def test_bin16_from_float(): - __run_test('f', 2**8, b'\xc5', b'\x01\x00', True) - __run_test('f', 2**16-4, b'\xc5', b'\xff\xfc', True) + _runtest('f', 2**8, b'\xc5', b'\x01\x00', True) + _runtest('f', 2**16-4, b'\xc5', b'\xff\xfc', True) def test_bin32_from_byte(): - __run_test('B', 2**16, b'\xc6', b'\x00\x01\x00\x00', True) + _runtest('B', 2**16, b'\xc6', b'\x00\x01\x00\x00', True) def test_bin32_from_float(): - __run_test('f', 2**16, b'\xc6', b'\x00\x01\x00\x00', True) + _runtest('f', 2**16, b'\xc6', b'\x00\x01\x00\x00', True) From 63e23d37f9f3646f0fc3b327ddf1f3e1f200baf5 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 5 May 2016 02:07:46 +0900 Subject: [PATCH 0874/1172] travis: Use docker to test 32bit environment (#189) * travis: testing matrix.include feature to use docker * Add test script for 32bit * Fix OverflowError in 32bit Environment --- .travis.yml | 38 +++++++++++++++++++++++++++----------- docker/runtests.sh | 14 ++++++++++++++ msgpack/_packer.pyx | 4 ++-- 3 files changed, 43 insertions(+), 13 deletions(-) create mode 100755 docker/runtests.sh diff --git a/.travis.yml b/.travis.yml index e7e9b63..b4396cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,23 +1,39 @@ sudo: false +language: python +python: 3.5 cache: directories: - $HOME/.cache/pip -language: python -python: - - 3.5 branches: - only: - - master + only: + - master env: - - TOXENV=py27-c,py33-c,py34-c,py35-c - - TOXENV=py27-pure,py33-pure,py34-pure,py35-pure - - TOXENV=pypy-pure,pypy3-pure + - TOXENV=py27-c,py33-c,py34-c,py35-c + - TOXENV=py27-pure,py33-pure,py34-pure,py35-pure + - TOXENV=pypy-pure,pypy3-pure + +matrix: + include: + - sudo: required + services: + - docker + env: + - DOCKER_IMAGE=quay.io/pypa/manylinux1_i686 + install: + - pip install -U pip + - pip install cython + - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx + - docker pull $DOCKER_IMAGE + script: + - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh install: - - pip install -U pip - - pip install tox cython - - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx + - pip install -U pip + - pip install tox cython + - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx script: tox + +# vim: sw=2 ts=2 diff --git a/docker/runtests.sh b/docker/runtests.sh new file mode 100755 index 0000000..0d74802 --- /dev/null +++ b/docker/runtests.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e -x + +for V in cp35-cp35m cp34-cp34m cp27-cp27m cp27-cp27mu; do + PYBIN=/opt/python/$V/bin + $PYBIN/python setup.py install + rm -rf build/ # Avoid lib build by narrow Python is used by wide python + $PYBIN/pip install pytest + pushd test # prevent importing msgpack package in current directory. + $PYBIN/python -c 'import sys; print(hex(sys.maxsize))' + $PYBIN/python -c 'from msgpack import _packer, _unpacker' + $PYBIN/py.test -v + popd +done diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index e07b194..3981f20 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -272,7 +272,7 @@ cdef class Packer(object): msgpack_pack_ext(&self.pk, typecode, len(data)) msgpack_pack_raw_body(&self.pk, data, len(data)) - def pack_array_header(self, size_t size): + def pack_array_header(self, long long size): if size > ITEM_LIMIT: raise PackValueError cdef int ret = msgpack_pack_array(&self.pk, size) @@ -285,7 +285,7 @@ cdef class Packer(object): self.pk.length = 0 return buf - def pack_map_header(self, size_t size): + def pack_map_header(self, long long size): if size > ITEM_LIMIT: raise PackValueError cdef int ret = msgpack_pack_map(&self.pk, size) From 5c052264bc52c4bbdb5e5736f4f24834af46b8d6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 5 May 2016 02:31:03 +0900 Subject: [PATCH 0875/1172] Update ChangeLog --- ChangeLog.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index f20bb75..396ccb7 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,29 @@ +0.5.0 +===== +:release date: TBD + +0.5 is important step toward 1.0. There are some deprecations. +Please read changes carefully. + +Changes +------- + +* Drop Python 2.6 and 3.2 support + +* Deprecate useless custom exceptions. Use ValueError instead of PackValueError, + Exception instead of PackException and UnpackException, etc... + See msgpack/exceptions.py + +* Add `strict_types` option to packer. It can be used to serialize subclass of + builtin types. For example, when packing object which type is subclass of dict, + `default()` is called. + +* Pure Python implementation supports packing memoryview object. + +Bugs fixed +---------- + + 0.4.7 ===== :release date: 2016-01-25 From a5c8bafad4ae29b0173f20dc1d7027219396f6aa Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 5 May 2016 02:46:10 +0900 Subject: [PATCH 0876/1172] Remove unused import (#190) --- msgpack/_packer.pyx | 3 --- 1 file changed, 3 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 3981f20..5c950ce 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -2,9 +2,6 @@ #cython: embedsignature=True from cpython cimport * -from libc.stdlib cimport * -from libc.string cimport * -from libc.limits cimport * from msgpack.exceptions import PackValueError, PackOverflowError from msgpack import ExtType From c6c4e59f4cb2ccb6de697de5d55a52e57f6a9a4e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 8 May 2016 16:31:52 +0900 Subject: [PATCH 0877/1172] s/realloc/PyMem_Realloc/ (#193) --- msgpack/pack.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/pack.h b/msgpack/pack.h index a75bdb0..d3aeff7 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -47,7 +47,7 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ if (len + l > bs) { bs = (len + l) * 2; - buf = (char*)realloc(buf, bs); + buf = (char*)PyMem_Realloc(buf, bs); if (!buf) return -1; } memcpy(buf + len, data, l); From 318ddfc0527ef3ddf1ad3467ece10c26efa8d741 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 13 May 2016 09:35:02 +0900 Subject: [PATCH 0878/1172] Remove wrong download_url from package metadata --- Makefile | 6 ++++++ setup.py | 7 +++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 2e53d08..01d8092 100644 --- a/Makefile +++ b/Makefile @@ -20,3 +20,9 @@ python3: cython test: py.test test + +.PHONY: clean +clean: + rm -rf build + rm msgpack/*.so + rm -rf msgpack/__pycache__ diff --git a/setup.py b/setup.py index 37729bd..d62c8f2 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # coding: utf-8 +import io import os import sys from glob import glob @@ -97,9 +98,8 @@ del libraries, macros desc = 'MessagePack (de)serializer.' -f = open('README.rst') -long_desc = f.read() -f.close() +with io.open('README.rst', encoding='utf-8') as f: + long_desc = f.read() del f setup(name='msgpack-python', @@ -112,7 +112,6 @@ setup(name='msgpack-python', description=desc, long_description=long_desc, url='http://msgpack.org/', - download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', From f421f59a287ae26b7fa83a0cac18650d0dd09c03 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 19 May 2016 22:35:42 +0900 Subject: [PATCH 0879/1172] fallback: Rewrite buffer from array of bytes to bytearray --- msgpack/fallback.py | 169 ++++++++++++++++++-------------------------- 1 file changed, 69 insertions(+), 100 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index abed3d9..181d7e2 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -86,11 +86,8 @@ def unpack(stream, **kwargs): Raises `ExtraData` when `packed` contains extra bytes. See :class:`Unpacker` for options. """ - unpacker = Unpacker(stream, **kwargs) - ret = unpacker._fb_unpack() - if unpacker._fb_got_extradata(): - raise ExtraData(ret, unpacker._fb_get_extradata()) - return ret + data = stream.read() + return unpackb(data, **kwargs) def unpackb(packed, **kwargs): @@ -121,7 +118,7 @@ class Unpacker(object): If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. :param int read_size: - Used as `file_like.read(read_size)`. (default: `min(1024**2, max_buffer_size)`) + Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) :param bool use_list: If true, unpack msgpack array to Python list. @@ -199,13 +196,9 @@ class Unpacker(object): self._fb_feeding = False #: array of bytes feeded. - self._fb_buffers = [] - #: Which buffer we currently reads - self._fb_buf_i = 0 + self._buffer = b"" #: Which position we currently reads - self._fb_buf_o = 0 - #: Total size of _fb_bufferes - self._fb_buf_n = 0 + self._buff_i = 0 # When Unpacker is used as an iterable, between the calls to next(), # the buffer is not "consumed" completely, for efficiency sake. @@ -213,13 +206,13 @@ class Unpacker(object): # the correct moments, we have to keep track of how sloppy we were. # Furthermore, when the buffer is incomplete (that is: in the case # we raise an OutOfData) we need to rollback the buffer to the correct - # state, which _fb_slopiness records. - self._fb_sloppiness = 0 + # state, which _buf_checkpoint records. + self._buf_checkpoint = 0 self._max_buffer_size = max_buffer_size or 2**31-1 if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") - self._read_size = read_size or min(self._max_buffer_size, 4096) + self._read_size = read_size or min(self._max_buffer_size, 16*1024) self._encoding = encoding self._unicode_errors = unicode_errors self._use_list = use_list @@ -248,103 +241,75 @@ class Unpacker(object): def feed(self, next_bytes): if isinstance(next_bytes, array.array): next_bytes = next_bytes.tostring() - elif isinstance(next_bytes, bytearray): - next_bytes = bytes(next_bytes) + if not isinstance(next_bytes, (bytes, bytearray)): + raise TypeError("next_bytes should be bytes, bytearray or array.array") assert self._fb_feeding - if (self._fb_buf_n + len(next_bytes) - self._fb_sloppiness - > self._max_buffer_size): - raise BufferFull - self._fb_buf_n += len(next_bytes) - self._fb_buffers.append(next_bytes) - def _fb_sloppy_consume(self): - """ Gets rid of some of the used parts of the buffer. """ - if self._fb_buf_i: - for i in xrange(self._fb_buf_i): - self._fb_buf_n -= len(self._fb_buffers[i]) - self._fb_buffers = self._fb_buffers[self._fb_buf_i:] - self._fb_buf_i = 0 - if self._fb_buffers: - self._fb_sloppiness = self._fb_buf_o - else: - self._fb_sloppiness = 0 + if (len(self._buffer) - self._buff_i + len(next_bytes) > self._max_buffer_size): + raise BufferFull + # bytes + bytearray -> bytearray + # So cast before append + self._buffer += bytes(next_bytes) def _fb_consume(self): """ Gets rid of the used parts of the buffer. """ - if self._fb_buf_i: - for i in xrange(self._fb_buf_i): - self._fb_buf_n -= len(self._fb_buffers[i]) - self._fb_buffers = self._fb_buffers[self._fb_buf_i:] - self._fb_buf_i = 0 - if self._fb_buffers: - self._fb_buffers[0] = self._fb_buffers[0][self._fb_buf_o:] - self._fb_buf_n -= self._fb_buf_o - else: - self._fb_buf_n = 0 - self._fb_buf_o = 0 - self._fb_sloppiness = 0 + self._buf_checkpoint = self._buff_i def _fb_got_extradata(self): - if self._fb_buf_i != len(self._fb_buffers): - return True - if self._fb_feeding: - return False - if not self.file_like: - return False - if self.file_like.read(1): - return True - return False + return self._buff_i < len(self._buffer) - def __iter__(self): - return self + def _fb_get_extradata(self): + return self._buffer[self._buff_i:] def read_bytes(self, n): return self._fb_read(n) - def _fb_rollback(self): - self._fb_buf_i = 0 - self._fb_buf_o = self._fb_sloppiness - - def _fb_get_extradata(self): - bufs = self._fb_buffers[self._fb_buf_i:] - if bufs: - bufs[0] = bufs[0][self._fb_buf_o:] - return b''.join(bufs) - def _fb_read(self, n, write_bytes=None): - buffs = self._fb_buffers - # We have a redundant codepath for the most common case, such that - # pypy optimizes it properly. This is the case that the read fits - # in the current buffer. - if (write_bytes is None and self._fb_buf_i < len(buffs) and - self._fb_buf_o + n < len(buffs[self._fb_buf_i])): - self._fb_buf_o += n - return buffs[self._fb_buf_i][self._fb_buf_o - n:self._fb_buf_o] + # (int, Optional[Callable]) -> bytearray + remain_bytes = len(self._buffer) - self._buff_i - n - # The remaining cases. - ret = b'' - while len(ret) != n: - sliced = n - len(ret) - if self._fb_buf_i == len(buffs): - if self._fb_feeding: - break - to_read = sliced - if self._read_size > to_read: - to_read = self._read_size - tmp = self.file_like.read(to_read) - if not tmp: - break - buffs.append(tmp) - self._fb_buf_n += len(tmp) - continue - ret += buffs[self._fb_buf_i][self._fb_buf_o:self._fb_buf_o + sliced] - self._fb_buf_o += sliced - if self._fb_buf_o >= len(buffs[self._fb_buf_i]): - self._fb_buf_o = 0 - self._fb_buf_i += 1 - if len(ret) != n: - self._fb_rollback() + # Fast path: buffer has n bytes already + if remain_bytes >= 0: + ret = self._buffer[self._buff_i:self._buff_i+n] + self._buff_i += n + if write_bytes is not None: + write_bytes(ret) + return ret + + if self._fb_feeding: + self._buff_i = self._buf_checkpoint raise OutOfData + + # Strip buffer before checkpoint before reading file. + if self._buf_checkpoint > 0: + self._buffer = self._buffer[self._buf_checkpoint:] + self._buff_i -= self._buf_checkpoint + self._buf_checkpoint = 0 + + # Read from file + remain_bytes = -remain_bytes + while remain_bytes > 0: + to_read_bytes = max(self._read_size, remain_bytes) + read_data = self.file_like.read(to_read_bytes) + if not read_data: + break + assert isinstance(read_data, bytes) + self._buffer += read_data + remain_bytes -= len(read_data) + + if len(self._buffer) < n + self._buff_i: + self._buff_i = 0 # rollback + raise OutOfData + + if len(self._buffer) == n: + # checkpoint == 0 + ret = self._buffer + self._buffer = b"" + self._buff_i = 0 + else: + ret = self._buffer[self._buff_i:self._buff_i+n] + self._buff_i += n + if write_bytes is not None: write_bytes(ret) return ret @@ -562,15 +527,19 @@ class Unpacker(object): assert typ == TYPE_IMMEDIATE return obj - def next(self): + def __iter__(self): + return self + + def __next__(self): try: ret = self._fb_unpack(EX_CONSTRUCT, None) - self._fb_sloppy_consume() + self._fb_consume() return ret except OutOfData: self._fb_consume() raise StopIteration - __next__ = next + + next = __next__ def skip(self, write_bytes=None): self._fb_unpack(EX_SKIP, write_bytes) From 3322a769890458ec8df5a365f9303510e59c3efb Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 22 May 2016 11:08:20 +0900 Subject: [PATCH 0880/1172] Remove _fb_ prefix --- msgpack/fallback.py | 172 ++++++++++++++++++++++---------------------- 1 file changed, 86 insertions(+), 86 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 181d7e2..95be713 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -100,11 +100,11 @@ def unpackb(packed, **kwargs): unpacker = Unpacker(None, **kwargs) unpacker.feed(packed) try: - ret = unpacker._fb_unpack() + ret = unpacker._unpack() except OutOfData: raise UnpackValueError("Data is not enough.") - if unpacker._fb_got_extradata(): - raise ExtraData(ret, unpacker._fb_get_extradata()) + if unpacker._got_extradata(): + raise ExtraData(ret, unpacker._get_extradata()) return ret @@ -188,12 +188,12 @@ class Unpacker(object): max_map_len=2147483647, max_ext_len=2147483647): if file_like is None: - self._fb_feeding = True + self._feeding = True else: if not callable(file_like.read): raise TypeError("`file_like.read` must be callable") self.file_like = file_like - self._fb_feeding = False + self._feeding = False #: array of bytes feeded. self._buffer = b"" @@ -243,7 +243,7 @@ class Unpacker(object): next_bytes = next_bytes.tostring() if not isinstance(next_bytes, (bytes, bytearray)): raise TypeError("next_bytes should be bytes, bytearray or array.array") - assert self._fb_feeding + assert self._feeding if (len(self._buffer) - self._buff_i + len(next_bytes) > self._max_buffer_size): raise BufferFull @@ -251,20 +251,20 @@ class Unpacker(object): # So cast before append self._buffer += bytes(next_bytes) - def _fb_consume(self): + def _consume(self): """ Gets rid of the used parts of the buffer. """ self._buf_checkpoint = self._buff_i - def _fb_got_extradata(self): + def _got_extradata(self): return self._buff_i < len(self._buffer) - def _fb_get_extradata(self): + def _get_extradata(self): return self._buffer[self._buff_i:] def read_bytes(self, n): - return self._fb_read(n) + return self._read(n) - def _fb_read(self, n, write_bytes=None): + def _read(self, n, write_bytes=None): # (int, Optional[Callable]) -> bytearray remain_bytes = len(self._buffer) - self._buff_i - n @@ -276,7 +276,7 @@ class Unpacker(object): write_bytes(ret) return ret - if self._fb_feeding: + if self._feeding: self._buff_i = self._buf_checkpoint raise OutOfData @@ -318,7 +318,7 @@ class Unpacker(object): typ = TYPE_IMMEDIATE n = 0 obj = None - c = self._fb_read(1, write_bytes) + c = self._read(1, write_bytes) b = ord(c) if b & 0b10000000 == 0: obj = b @@ -326,7 +326,7 @@ class Unpacker(object): obj = struct.unpack("b", c)[0] elif b & 0b11100000 == 0b10100000: n = b & 0b00011111 - obj = self._fb_read(n, write_bytes) + obj = self._read(n, write_bytes) typ = TYPE_RAW if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) @@ -348,120 +348,120 @@ class Unpacker(object): obj = True elif b == 0xc4: typ = TYPE_BIN - n = struct.unpack("B", self._fb_read(1, write_bytes))[0] + n = struct.unpack("B", self._read(1, write_bytes))[0] if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) - obj = self._fb_read(n, write_bytes) + obj = self._read(n, write_bytes) elif b == 0xc5: typ = TYPE_BIN - n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + n = struct.unpack(">H", self._read(2, write_bytes))[0] if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) - obj = self._fb_read(n, write_bytes) + obj = self._read(n, write_bytes) elif b == 0xc6: typ = TYPE_BIN - n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + n = struct.unpack(">I", self._read(4, write_bytes))[0] if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) - obj = self._fb_read(n, write_bytes) + obj = self._read(n, write_bytes) elif b == 0xc7: # ext 8 typ = TYPE_EXT - L, n = struct.unpack('Bb', self._fb_read(2, write_bytes)) + L, n = struct.unpack('Bb', self._read(2, write_bytes)) if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) - obj = self._fb_read(L, write_bytes) + obj = self._read(L, write_bytes) elif b == 0xc8: # ext 16 typ = TYPE_EXT - L, n = struct.unpack('>Hb', self._fb_read(3, write_bytes)) + L, n = struct.unpack('>Hb', self._read(3, write_bytes)) if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) - obj = self._fb_read(L, write_bytes) + obj = self._read(L, write_bytes) elif b == 0xc9: # ext 32 typ = TYPE_EXT - L, n = struct.unpack('>Ib', self._fb_read(5, write_bytes)) + L, n = struct.unpack('>Ib', self._read(5, write_bytes)) if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) - obj = self._fb_read(L, write_bytes) + obj = self._read(L, write_bytes) elif b == 0xca: - obj = struct.unpack(">f", self._fb_read(4, write_bytes))[0] + obj = struct.unpack(">f", self._read(4, write_bytes))[0] elif b == 0xcb: - obj = struct.unpack(">d", self._fb_read(8, write_bytes))[0] + obj = struct.unpack(">d", self._read(8, write_bytes))[0] elif b == 0xcc: - obj = struct.unpack("B", self._fb_read(1, write_bytes))[0] + obj = struct.unpack("B", self._read(1, write_bytes))[0] elif b == 0xcd: - obj = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + obj = struct.unpack(">H", self._read(2, write_bytes))[0] elif b == 0xce: - obj = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + obj = struct.unpack(">I", self._read(4, write_bytes))[0] elif b == 0xcf: - obj = struct.unpack(">Q", self._fb_read(8, write_bytes))[0] + obj = struct.unpack(">Q", self._read(8, write_bytes))[0] elif b == 0xd0: - obj = struct.unpack("b", self._fb_read(1, write_bytes))[0] + obj = struct.unpack("b", self._read(1, write_bytes))[0] elif b == 0xd1: - obj = struct.unpack(">h", self._fb_read(2, write_bytes))[0] + obj = struct.unpack(">h", self._read(2, write_bytes))[0] elif b == 0xd2: - obj = struct.unpack(">i", self._fb_read(4, write_bytes))[0] + obj = struct.unpack(">i", self._read(4, write_bytes))[0] elif b == 0xd3: - obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0] + obj = struct.unpack(">q", self._read(8, write_bytes))[0] elif b == 0xd4: # fixext 1 typ = TYPE_EXT if self._max_ext_len < 1: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) - n, obj = struct.unpack('b1s', self._fb_read(2, write_bytes)) + n, obj = struct.unpack('b1s', self._read(2, write_bytes)) elif b == 0xd5: # fixext 2 typ = TYPE_EXT if self._max_ext_len < 2: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) - n, obj = struct.unpack('b2s', self._fb_read(3, write_bytes)) + n, obj = struct.unpack('b2s', self._read(3, write_bytes)) elif b == 0xd6: # fixext 4 typ = TYPE_EXT if self._max_ext_len < 4: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) - n, obj = struct.unpack('b4s', self._fb_read(5, write_bytes)) + n, obj = struct.unpack('b4s', self._read(5, write_bytes)) elif b == 0xd7: # fixext 8 typ = TYPE_EXT if self._max_ext_len < 8: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) - n, obj = struct.unpack('b8s', self._fb_read(9, write_bytes)) + n, obj = struct.unpack('b8s', self._read(9, write_bytes)) elif b == 0xd8: # fixext 16 typ = TYPE_EXT if self._max_ext_len < 16: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) - n, obj = struct.unpack('b16s', self._fb_read(17, write_bytes)) + n, obj = struct.unpack('b16s', self._read(17, write_bytes)) elif b == 0xd9: typ = TYPE_RAW - n = struct.unpack("B", self._fb_read(1, write_bytes))[0] + n = struct.unpack("B", self._read(1, write_bytes))[0] if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._fb_read(n, write_bytes) + obj = self._read(n, write_bytes) elif b == 0xda: typ = TYPE_RAW - n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + n = struct.unpack(">H", self._read(2, write_bytes))[0] if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._fb_read(n, write_bytes) + obj = self._read(n, write_bytes) elif b == 0xdb: typ = TYPE_RAW - n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + n = struct.unpack(">I", self._read(4, write_bytes))[0] if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._fb_read(n, write_bytes) + obj = self._read(n, write_bytes) elif b == 0xdc: - n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + n = struct.unpack(">H", self._read(2, write_bytes))[0] if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) typ = TYPE_ARRAY elif b == 0xdd: - n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + n = struct.unpack(">I", self._read(4, write_bytes))[0] if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) typ = TYPE_ARRAY elif b == 0xde: - n = struct.unpack(">H", self._fb_read(2, write_bytes))[0] + n = struct.unpack(">H", self._read(2, write_bytes))[0] if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP elif b == 0xdf: - n = struct.unpack(">I", self._fb_read(4, write_bytes))[0] + n = struct.unpack(">I", self._read(4, write_bytes))[0] if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP @@ -469,7 +469,7 @@ class Unpacker(object): raise UnpackValueError("Unknown header: 0x%x" % b) return typ, n, obj - def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None): + def _unpack(self, execute=EX_CONSTRUCT, write_bytes=None): typ, n, obj = self._read_header(execute, write_bytes) if execute == EX_READ_ARRAY_HEADER: @@ -485,11 +485,11 @@ class Unpacker(object): if execute == EX_SKIP: for i in xrange(n): # TODO check whether we need to call `list_hook` - self._fb_unpack(EX_SKIP, write_bytes) + self._unpack(EX_SKIP, write_bytes) return ret = newlist_hint(n) for i in xrange(n): - ret.append(self._fb_unpack(EX_CONSTRUCT, write_bytes)) + ret.append(self._unpack(EX_CONSTRUCT, write_bytes)) if self._list_hook is not None: ret = self._list_hook(ret) # TODO is the interaction between `list_hook` and `use_list` ok? @@ -498,19 +498,19 @@ class Unpacker(object): if execute == EX_SKIP: for i in xrange(n): # TODO check whether we need to call hooks - self._fb_unpack(EX_SKIP, write_bytes) - self._fb_unpack(EX_SKIP, write_bytes) + self._unpack(EX_SKIP, write_bytes) + self._unpack(EX_SKIP, write_bytes) return if self._object_pairs_hook is not None: ret = self._object_pairs_hook( - (self._fb_unpack(EX_CONSTRUCT, write_bytes), - self._fb_unpack(EX_CONSTRUCT, write_bytes)) + (self._unpack(EX_CONSTRUCT, write_bytes), + self._unpack(EX_CONSTRUCT, write_bytes)) for _ in xrange(n)) else: ret = {} for _ in xrange(n): - key = self._fb_unpack(EX_CONSTRUCT, write_bytes) - ret[key] = self._fb_unpack(EX_CONSTRUCT, write_bytes) + key = self._unpack(EX_CONSTRUCT, write_bytes) + ret[key] = self._unpack(EX_CONSTRUCT, write_bytes) if self._object_hook is not None: ret = self._object_hook(ret) return ret @@ -532,32 +532,32 @@ class Unpacker(object): def __next__(self): try: - ret = self._fb_unpack(EX_CONSTRUCT, None) - self._fb_consume() + ret = self._unpack(EX_CONSTRUCT, None) + self._consume() return ret except OutOfData: - self._fb_consume() + self._consume() raise StopIteration next = __next__ def skip(self, write_bytes=None): - self._fb_unpack(EX_SKIP, write_bytes) - self._fb_consume() + self._unpack(EX_SKIP, write_bytes) + self._consume() def unpack(self, write_bytes=None): - ret = self._fb_unpack(EX_CONSTRUCT, write_bytes) - self._fb_consume() + ret = self._unpack(EX_CONSTRUCT, write_bytes) + self._consume() return ret def read_array_header(self, write_bytes=None): - ret = self._fb_unpack(EX_READ_ARRAY_HEADER, write_bytes) - self._fb_consume() + ret = self._unpack(EX_READ_ARRAY_HEADER, write_bytes) + self._consume() return ret def read_map_header(self, write_bytes=None): - ret = self._fb_unpack(EX_READ_MAP_HEADER, write_bytes) - self._fb_consume() + ret = self._unpack(EX_READ_MAP_HEADER, write_bytes) + self._consume() return ret @@ -658,7 +658,7 @@ class Packer(object): n = len(obj) if n >= 2**32: raise PackValueError("Bytes is too large") - self._fb_pack_bin_header(n) + self._pack_bin_header(n) return self._buffer.write(obj) if check(obj, Unicode): if self._encoding is None: @@ -669,13 +669,13 @@ class Packer(object): n = len(obj) if n >= 2**32: raise PackValueError("String is too large") - self._fb_pack_raw_header(n) + self._pack_raw_header(n) return self._buffer.write(obj) if check(obj, memoryview): n = len(obj) * obj.itemsize if n >= 2**32: raise PackValueError("Memoryview is too large") - self._fb_pack_bin_header(n) + self._pack_bin_header(n) return self._buffer.write(obj) if check(obj, float): if self._use_float: @@ -708,12 +708,12 @@ class Packer(object): return if check(obj, list_types): n = len(obj) - self._fb_pack_array_header(n) + self._pack_array_header(n) for i in xrange(n): self._pack(obj[i], nest_limit - 1) return if check(obj, dict): - return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj), + return self._pack_map_pairs(len(obj), dict_iteritems(obj), nest_limit - 1) if not default_used and self._default is not None: obj = self._default(obj) @@ -731,7 +731,7 @@ class Packer(object): return ret def pack_map_pairs(self, pairs): - self._fb_pack_map_pairs(len(pairs), pairs) + self._pack_map_pairs(len(pairs), pairs) ret = self._buffer.getvalue() if self._autoreset: self._buffer = StringIO() @@ -742,7 +742,7 @@ class Packer(object): def pack_array_header(self, n): if n >= 2**32: raise PackValueError - self._fb_pack_array_header(n) + self._pack_array_header(n) ret = self._buffer.getvalue() if self._autoreset: self._buffer = StringIO() @@ -753,7 +753,7 @@ class Packer(object): def pack_map_header(self, n): if n >= 2**32: raise PackValueError - self._fb_pack_map_header(n) + self._pack_map_header(n) ret = self._buffer.getvalue() if self._autoreset: self._buffer = StringIO() @@ -790,7 +790,7 @@ class Packer(object): self._buffer.write(struct.pack('B', typecode)) self._buffer.write(data) - def _fb_pack_array_header(self, n): + def _pack_array_header(self, n): if n <= 0x0f: return self._buffer.write(struct.pack('B', 0x90 + n)) if n <= 0xffff: @@ -799,7 +799,7 @@ class Packer(object): return self._buffer.write(struct.pack(">BI", 0xdd, n)) raise PackValueError("Array is too large") - def _fb_pack_map_header(self, n): + def _pack_map_header(self, n): if n <= 0x0f: return self._buffer.write(struct.pack('B', 0x80 + n)) if n <= 0xffff: @@ -808,13 +808,13 @@ class Packer(object): return self._buffer.write(struct.pack(">BI", 0xdf, n)) raise PackValueError("Dict is too large") - def _fb_pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT): - self._fb_pack_map_header(n) + def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT): + self._pack_map_header(n) for (k, v) in pairs: self._pack(k, nest_limit - 1) self._pack(v, nest_limit - 1) - def _fb_pack_raw_header(self, n): + def _pack_raw_header(self, n): if n <= 0x1f: self._buffer.write(struct.pack('B', 0xa0 + n)) elif self._use_bin_type and n <= 0xff: @@ -826,9 +826,9 @@ class Packer(object): else: raise PackValueError('Raw is too large') - def _fb_pack_bin_header(self, n): + def _pack_bin_header(self, n): if not self._use_bin_type: - return self._fb_pack_raw_header(n) + return self._pack_raw_header(n) elif n <= 0xff: return self._buffer.write(struct.pack('>BB', 0xc4, n)) elif n <= 0xffff: From e9c42fa523b51c184b581d6eab85f0ad40dff620 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 22 May 2016 13:31:01 +0900 Subject: [PATCH 0881/1172] fallback: simplify write_bytes callback implementation --- msgpack/fallback.py | 124 +++++++++++++++++++++++--------------------- 1 file changed, 64 insertions(+), 60 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 95be713..ecdbec4 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -264,7 +264,7 @@ class Unpacker(object): def read_bytes(self, n): return self._read(n) - def _read(self, n, write_bytes=None): + def _read(self, n): # (int, Optional[Callable]) -> bytearray remain_bytes = len(self._buffer) - self._buff_i - n @@ -272,8 +272,6 @@ class Unpacker(object): if remain_bytes >= 0: ret = self._buffer[self._buff_i:self._buff_i+n] self._buff_i += n - if write_bytes is not None: - write_bytes(ret) return ret if self._feeding: @@ -310,15 +308,13 @@ class Unpacker(object): ret = self._buffer[self._buff_i:self._buff_i+n] self._buff_i += n - if write_bytes is not None: - write_bytes(ret) return ret - def _read_header(self, execute=EX_CONSTRUCT, write_bytes=None): + def _read_header(self, execute=EX_CONSTRUCT): typ = TYPE_IMMEDIATE n = 0 obj = None - c = self._read(1, write_bytes) + c = self._read(1) b = ord(c) if b & 0b10000000 == 0: obj = b @@ -326,7 +322,7 @@ class Unpacker(object): obj = struct.unpack("b", c)[0] elif b & 0b11100000 == 0b10100000: n = b & 0b00011111 - obj = self._read(n, write_bytes) + obj = self._read(n) typ = TYPE_RAW if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) @@ -348,120 +344,120 @@ class Unpacker(object): obj = True elif b == 0xc4: typ = TYPE_BIN - n = struct.unpack("B", self._read(1, write_bytes))[0] + n = struct.unpack("B", self._read(1))[0] if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) - obj = self._read(n, write_bytes) + obj = self._read(n) elif b == 0xc5: typ = TYPE_BIN - n = struct.unpack(">H", self._read(2, write_bytes))[0] + n = struct.unpack(">H", self._read(2))[0] if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) - obj = self._read(n, write_bytes) + obj = self._read(n) elif b == 0xc6: typ = TYPE_BIN - n = struct.unpack(">I", self._read(4, write_bytes))[0] + n = struct.unpack(">I", self._read(4))[0] if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) - obj = self._read(n, write_bytes) + obj = self._read(n) elif b == 0xc7: # ext 8 typ = TYPE_EXT - L, n = struct.unpack('Bb', self._read(2, write_bytes)) + L, n = struct.unpack('Bb', self._read(2)) if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) - obj = self._read(L, write_bytes) + obj = self._read(L) elif b == 0xc8: # ext 16 typ = TYPE_EXT - L, n = struct.unpack('>Hb', self._read(3, write_bytes)) + L, n = struct.unpack('>Hb', self._read(3)) if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) - obj = self._read(L, write_bytes) + obj = self._read(L) elif b == 0xc9: # ext 32 typ = TYPE_EXT - L, n = struct.unpack('>Ib', self._read(5, write_bytes)) + L, n = struct.unpack('>Ib', self._read(5)) if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) - obj = self._read(L, write_bytes) + obj = self._read(L) elif b == 0xca: - obj = struct.unpack(">f", self._read(4, write_bytes))[0] + obj = struct.unpack(">f", self._read(4))[0] elif b == 0xcb: - obj = struct.unpack(">d", self._read(8, write_bytes))[0] + obj = struct.unpack(">d", self._read(8))[0] elif b == 0xcc: - obj = struct.unpack("B", self._read(1, write_bytes))[0] + obj = struct.unpack("B", self._read(1))[0] elif b == 0xcd: - obj = struct.unpack(">H", self._read(2, write_bytes))[0] + obj = struct.unpack(">H", self._read(2))[0] elif b == 0xce: - obj = struct.unpack(">I", self._read(4, write_bytes))[0] + obj = struct.unpack(">I", self._read(4))[0] elif b == 0xcf: - obj = struct.unpack(">Q", self._read(8, write_bytes))[0] + obj = struct.unpack(">Q", self._read(8))[0] elif b == 0xd0: - obj = struct.unpack("b", self._read(1, write_bytes))[0] + obj = struct.unpack("b", self._read(1))[0] elif b == 0xd1: - obj = struct.unpack(">h", self._read(2, write_bytes))[0] + obj = struct.unpack(">h", self._read(2))[0] elif b == 0xd2: - obj = struct.unpack(">i", self._read(4, write_bytes))[0] + obj = struct.unpack(">i", self._read(4))[0] elif b == 0xd3: - obj = struct.unpack(">q", self._read(8, write_bytes))[0] + obj = struct.unpack(">q", self._read(8))[0] elif b == 0xd4: # fixext 1 typ = TYPE_EXT if self._max_ext_len < 1: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) - n, obj = struct.unpack('b1s', self._read(2, write_bytes)) + n, obj = struct.unpack('b1s', self._read(2)) elif b == 0xd5: # fixext 2 typ = TYPE_EXT if self._max_ext_len < 2: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) - n, obj = struct.unpack('b2s', self._read(3, write_bytes)) + n, obj = struct.unpack('b2s', self._read(3)) elif b == 0xd6: # fixext 4 typ = TYPE_EXT if self._max_ext_len < 4: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) - n, obj = struct.unpack('b4s', self._read(5, write_bytes)) + n, obj = struct.unpack('b4s', self._read(5)) elif b == 0xd7: # fixext 8 typ = TYPE_EXT if self._max_ext_len < 8: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) - n, obj = struct.unpack('b8s', self._read(9, write_bytes)) + n, obj = struct.unpack('b8s', self._read(9)) elif b == 0xd8: # fixext 16 typ = TYPE_EXT if self._max_ext_len < 16: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) - n, obj = struct.unpack('b16s', self._read(17, write_bytes)) + n, obj = struct.unpack('b16s', self._read(17)) elif b == 0xd9: typ = TYPE_RAW - n = struct.unpack("B", self._read(1, write_bytes))[0] + n = struct.unpack("B", self._read(1))[0] if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._read(n, write_bytes) + obj = self._read(n) elif b == 0xda: typ = TYPE_RAW - n = struct.unpack(">H", self._read(2, write_bytes))[0] + n = struct.unpack(">H", self._read(2))[0] if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._read(n, write_bytes) + obj = self._read(n) elif b == 0xdb: typ = TYPE_RAW - n = struct.unpack(">I", self._read(4, write_bytes))[0] + n = struct.unpack(">I", self._read(4))[0] if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._read(n, write_bytes) + obj = self._read(n) elif b == 0xdc: - n = struct.unpack(">H", self._read(2, write_bytes))[0] + n = struct.unpack(">H", self._read(2))[0] if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) typ = TYPE_ARRAY elif b == 0xdd: - n = struct.unpack(">I", self._read(4, write_bytes))[0] + n = struct.unpack(">I", self._read(4))[0] if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) typ = TYPE_ARRAY elif b == 0xde: - n = struct.unpack(">H", self._read(2, write_bytes))[0] + n = struct.unpack(">H", self._read(2))[0] if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP elif b == 0xdf: - n = struct.unpack(">I", self._read(4, write_bytes))[0] + n = struct.unpack(">I", self._read(4))[0] if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP @@ -469,8 +465,8 @@ class Unpacker(object): raise UnpackValueError("Unknown header: 0x%x" % b) return typ, n, obj - def _unpack(self, execute=EX_CONSTRUCT, write_bytes=None): - typ, n, obj = self._read_header(execute, write_bytes) + def _unpack(self, execute=EX_CONSTRUCT): + typ, n, obj = self._read_header(execute) if execute == EX_READ_ARRAY_HEADER: if typ != TYPE_ARRAY: @@ -485,11 +481,11 @@ class Unpacker(object): if execute == EX_SKIP: for i in xrange(n): # TODO check whether we need to call `list_hook` - self._unpack(EX_SKIP, write_bytes) + self._unpack(EX_SKIP) return ret = newlist_hint(n) for i in xrange(n): - ret.append(self._unpack(EX_CONSTRUCT, write_bytes)) + ret.append(self._unpack(EX_CONSTRUCT)) if self._list_hook is not None: ret = self._list_hook(ret) # TODO is the interaction between `list_hook` and `use_list` ok? @@ -498,19 +494,19 @@ class Unpacker(object): if execute == EX_SKIP: for i in xrange(n): # TODO check whether we need to call hooks - self._unpack(EX_SKIP, write_bytes) - self._unpack(EX_SKIP, write_bytes) + self._unpack(EX_SKIP) + self._unpack(EX_SKIP) return if self._object_pairs_hook is not None: ret = self._object_pairs_hook( - (self._unpack(EX_CONSTRUCT, write_bytes), - self._unpack(EX_CONSTRUCT, write_bytes)) + (self._unpack(EX_CONSTRUCT), + self._unpack(EX_CONSTRUCT)) for _ in xrange(n)) else: ret = {} for _ in xrange(n): - key = self._unpack(EX_CONSTRUCT, write_bytes) - ret[key] = self._unpack(EX_CONSTRUCT, write_bytes) + key = self._unpack(EX_CONSTRUCT) + ret[key] = self._unpack(EX_CONSTRUCT) if self._object_hook is not None: ret = self._object_hook(ret) return ret @@ -532,7 +528,7 @@ class Unpacker(object): def __next__(self): try: - ret = self._unpack(EX_CONSTRUCT, None) + ret = self._unpack(EX_CONSTRUCT) self._consume() return ret except OutOfData: @@ -542,21 +538,29 @@ class Unpacker(object): next = __next__ def skip(self, write_bytes=None): - self._unpack(EX_SKIP, write_bytes) + self._unpack(EX_SKIP) + if write_bytes is not None: + write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) self._consume() def unpack(self, write_bytes=None): - ret = self._unpack(EX_CONSTRUCT, write_bytes) + ret = self._unpack(EX_CONSTRUCT) + if write_bytes is not None: + write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) self._consume() return ret def read_array_header(self, write_bytes=None): - ret = self._unpack(EX_READ_ARRAY_HEADER, write_bytes) + ret = self._unpack(EX_READ_ARRAY_HEADER) + if write_bytes is not None: + write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) self._consume() return ret def read_map_header(self, write_bytes=None): - ret = self._unpack(EX_READ_MAP_HEADER, write_bytes) + ret = self._unpack(EX_READ_MAP_HEADER) + if write_bytes is not None: + write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) self._consume() return ret From 6b8919355d6acdda74acc568dd348598e552f003 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 24 May 2016 02:46:29 +0900 Subject: [PATCH 0882/1172] fallback: Use struct.unpack_from when possible --- msgpack/fallback.py | 150 +++++++++++++++++++++++++++++--------------- 1 file changed, 100 insertions(+), 50 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index ecdbec4..9b32610 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -265,14 +265,18 @@ class Unpacker(object): return self._read(n) def _read(self, n): - # (int, Optional[Callable]) -> bytearray + # (int) -> bytearray + self._reserve(n) + i = self._buff_i + self._buff_i = i+n + return self._buffer[i:i+n] + + def _reserve(self, n): remain_bytes = len(self._buffer) - self._buff_i - n # Fast path: buffer has n bytes already if remain_bytes >= 0: - ret = self._buffer[self._buff_i:self._buff_i+n] - self._buff_i += n - return ret + return if self._feeding: self._buff_i = self._buf_checkpoint @@ -299,33 +303,23 @@ class Unpacker(object): self._buff_i = 0 # rollback raise OutOfData - if len(self._buffer) == n: - # checkpoint == 0 - ret = self._buffer - self._buffer = b"" - self._buff_i = 0 - else: - ret = self._buffer[self._buff_i:self._buff_i+n] - self._buff_i += n - - return ret - def _read_header(self, execute=EX_CONSTRUCT): typ = TYPE_IMMEDIATE n = 0 obj = None - c = self._read(1) - b = ord(c) - if b & 0b10000000 == 0: + self._reserve(1) + b = struct.unpack_from("B", self._buffer, self._buff_i)[0] + self._buff_i += 1 + if b & 0b10000000 == 0: obj = b elif b & 0b11100000 == 0b11100000: - obj = struct.unpack("b", c)[0] + obj = -1 - (b ^ 0xff) elif b & 0b11100000 == 0b10100000: n = b & 0b00011111 - obj = self._read(n) typ = TYPE_RAW if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + obj = self._read(n) elif b & 0b11110000 == 0b10010000: n = b & 0b00001111 typ = TYPE_ARRAY @@ -344,120 +338,176 @@ class Unpacker(object): obj = True elif b == 0xc4: typ = TYPE_BIN - n = struct.unpack("B", self._read(1))[0] + self._reserve(1) + n = struct.unpack_from("B", self._buffer, self._buff_i)[0] + self._buff_i += 1 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._read(n) elif b == 0xc5: typ = TYPE_BIN - n = struct.unpack(">H", self._read(2))[0] + self._reserve(2) + n = struct.unpack_from(">H", self._buffer, self._buff_i)[0] + self._buff_i += 2 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._read(n) elif b == 0xc6: typ = TYPE_BIN - n = struct.unpack(">I", self._read(4))[0] + self._reserve(4) + n = struct.unpack_from(">I", self._buffer, self._buff_i)[0] + self._buff_i += 4 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._read(n) elif b == 0xc7: # ext 8 typ = TYPE_EXT - L, n = struct.unpack('Bb', self._read(2)) + self._reserve(2) + L, n = struct.unpack_from('Bb', self._buffer, self._buff_i) + self._buff_i += 2 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) elif b == 0xc8: # ext 16 typ = TYPE_EXT - L, n = struct.unpack('>Hb', self._read(3)) + self._reserve(3) + L, n = struct.unpack_from('>Hb', self._buffer, self._buff_i) + self._buff_i += 3 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) elif b == 0xc9: # ext 32 typ = TYPE_EXT - L, n = struct.unpack('>Ib', self._read(5)) + self._reserve(5) + L, n = struct.unpack_from('>Ib', self._buffer, self._buff_i) + self._buff_i += 5 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) elif b == 0xca: - obj = struct.unpack(">f", self._read(4))[0] + self._reserve(4) + obj = struct.unpack_from(">f", self._buffer, self._buff_i)[0] + self._buff_i += 4 elif b == 0xcb: - obj = struct.unpack(">d", self._read(8))[0] + self._reserve(8) + obj = struct.unpack_from(">d", self._buffer, self._buff_i)[0] + self._buff_i += 8 elif b == 0xcc: - obj = struct.unpack("B", self._read(1))[0] + self._reserve(1) + obj = struct.unpack_from("B", self._buffer, self._buff_i)[0] + self._buff_i += 1 elif b == 0xcd: - obj = struct.unpack(">H", self._read(2))[0] + self._reserve(2) + obj = struct.unpack_from(">H", self._buffer, self._buff_i)[0] + self._buff_i += 2 elif b == 0xce: - obj = struct.unpack(">I", self._read(4))[0] + self._reserve(4) + obj = struct.unpack_from(">I", self._buffer, self._buff_i)[0] + self._buff_i += 4 elif b == 0xcf: - obj = struct.unpack(">Q", self._read(8))[0] + self._reserve(8) + obj = struct.unpack_from(">Q", self._buffer, self._buff_i)[0] + self._buff_i += 8 elif b == 0xd0: - obj = struct.unpack("b", self._read(1))[0] + self._reserve(1) + obj = struct.unpack_from("b", self._buffer, self._buff_i)[0] + self._buff_i += 1 elif b == 0xd1: - obj = struct.unpack(">h", self._read(2))[0] + self._reserve(2) + obj = struct.unpack_from(">h", self._buffer, self._buff_i)[0] + self._buff_i += 2 elif b == 0xd2: - obj = struct.unpack(">i", self._read(4))[0] + self._reserve(4) + obj = struct.unpack_from(">i", self._buffer, self._buff_i)[0] + self._buff_i += 4 elif b == 0xd3: - obj = struct.unpack(">q", self._read(8))[0] + self._reserve(8) + obj = struct.unpack_from(">q", self._buffer, self._buff_i)[0] + self._buff_i += 8 elif b == 0xd4: # fixext 1 typ = TYPE_EXT if self._max_ext_len < 1: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) - n, obj = struct.unpack('b1s', self._read(2)) + self._reserve(2) + n, obj = struct.unpack_from("b1s", self._buffer, self._buff_i) + self._buff_i += 2 elif b == 0xd5: # fixext 2 typ = TYPE_EXT if self._max_ext_len < 2: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) - n, obj = struct.unpack('b2s', self._read(3)) + self._reserve(3) + n, obj = struct.unpack_from("b2s", self._buffer, self._buff_i) + self._buff_i += 3 elif b == 0xd6: # fixext 4 typ = TYPE_EXT if self._max_ext_len < 4: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) - n, obj = struct.unpack('b4s', self._read(5)) + self._reserve(5) + n, obj = struct.unpack_from("b4s", self._buffer, self._buff_i) + self._buff_i += 5 elif b == 0xd7: # fixext 8 typ = TYPE_EXT if self._max_ext_len < 8: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) - n, obj = struct.unpack('b8s', self._read(9)) + self._reserve(9) + n, obj = struct.unpack_from("b8s", self._buffer, self._buff_i) + self._buff_i += 9 elif b == 0xd8: # fixext 16 typ = TYPE_EXT if self._max_ext_len < 16: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) - n, obj = struct.unpack('b16s', self._read(17)) + self._reserve(17) + n, obj = struct.unpack_from("b16s", self._buffer, self._buff_i) + self._buff_i += 17 elif b == 0xd9: typ = TYPE_RAW - n = struct.unpack("B", self._read(1))[0] + self._reserve(1) + n, = struct.unpack_from("B", self._buffer, self._buff_i) + self._buff_i += 1 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) elif b == 0xda: typ = TYPE_RAW - n = struct.unpack(">H", self._read(2))[0] + self._reserve(2) + n, = struct.unpack_from(">H", self._buffer, self._buff_i) + self._buff_i += 2 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) elif b == 0xdb: typ = TYPE_RAW - n = struct.unpack(">I", self._read(4))[0] + self._reserve(4) + n, = struct.unpack_from(">I", self._buffer, self._buff_i) + self._buff_i += 4 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) elif b == 0xdc: - n = struct.unpack(">H", self._read(2))[0] + typ = TYPE_ARRAY + self._reserve(2) + n, = struct.unpack_from(">H", self._buffer, self._buff_i) + self._buff_i += 2 if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) - typ = TYPE_ARRAY elif b == 0xdd: - n = struct.unpack(">I", self._read(4))[0] + typ = TYPE_ARRAY + self._reserve(4) + n, = struct.unpack_from(">I", self._buffer, self._buff_i) + self._buff_i += 4 if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) - typ = TYPE_ARRAY elif b == 0xde: - n = struct.unpack(">H", self._read(2))[0] + self._reserve(2) + n, = struct.unpack_from(">H", self._buffer, self._buff_i) + self._buff_i += 2 if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP elif b == 0xdf: - n = struct.unpack(">I", self._read(4))[0] + self._reserve(4) + n, = struct.unpack_from(">I", self._buffer, self._buff_i) + self._buff_i += 4 if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP From c16a1c6bdf667d5cfe314d09d5613808f1243a8f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 24 May 2016 07:32:30 +0900 Subject: [PATCH 0883/1172] fallback: Use bytearray as buffer --- msgpack/fallback.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 9b32610..a23ad8c 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -196,7 +196,7 @@ class Unpacker(object): self._feeding = False #: array of bytes feeded. - self._buffer = b"" + self._buffer = bytearray() #: Which position we currently reads self._buff_i = 0 @@ -249,7 +249,7 @@ class Unpacker(object): raise BufferFull # bytes + bytearray -> bytearray # So cast before append - self._buffer += bytes(next_bytes) + self._buffer += next_bytes def _consume(self): """ Gets rid of the used parts of the buffer. """ @@ -284,7 +284,7 @@ class Unpacker(object): # Strip buffer before checkpoint before reading file. if self._buf_checkpoint > 0: - self._buffer = self._buffer[self._buf_checkpoint:] + del self._buffer[:self._buf_checkpoint] self._buff_i -= self._buf_checkpoint self._buf_checkpoint = 0 @@ -308,7 +308,8 @@ class Unpacker(object): n = 0 obj = None self._reserve(1) - b = struct.unpack_from("B", self._buffer, self._buff_i)[0] + #b = struct.unpack_from("B", self._buffer, self._buff_i)[0] + b = self._buffer[self._buff_i] self._buff_i += 1 if b & 0b10000000 == 0: obj = b @@ -339,7 +340,8 @@ class Unpacker(object): elif b == 0xc4: typ = TYPE_BIN self._reserve(1) - n = struct.unpack_from("B", self._buffer, self._buff_i)[0] + #n = struct.unpack_from("B", self._buffer, self._buff_i)[0] + n = self._buffer[self._buff_i] self._buff_i += 1 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) @@ -394,7 +396,8 @@ class Unpacker(object): self._buff_i += 8 elif b == 0xcc: self._reserve(1) - obj = struct.unpack_from("B", self._buffer, self._buff_i)[0] + #obj = struct.unpack_from("B", self._buffer, self._buff_i)[0] + obj = self._buffer[self._buff_i] self._buff_i += 1 elif b == 0xcd: self._reserve(2) @@ -462,7 +465,8 @@ class Unpacker(object): elif b == 0xd9: typ = TYPE_RAW self._reserve(1) - n, = struct.unpack_from("B", self._buffer, self._buff_i) + #n, = struct.unpack_from("B", self._buffer, self._buff_i) + n = self._buffer[self._buff_i] self._buff_i += 1 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) @@ -565,11 +569,13 @@ class Unpacker(object): if typ == TYPE_RAW: if self._encoding is not None: obj = obj.decode(self._encoding, self._unicode_errors) + else: + obj = bytes(obj) return obj if typ == TYPE_EXT: - return self._ext_hook(n, obj) + return self._ext_hook(n, bytes(obj)) if typ == TYPE_BIN: - return obj + return bytes(obj) assert typ == TYPE_IMMEDIATE return obj From 2b63e9fbbb3440d73d6638ec8af6315aeb8ecd97 Mon Sep 17 00:00:00 2001 From: folz Date: Sat, 7 May 2016 15:18:20 +0200 Subject: [PATCH 0884/1172] enable unpacking from memoryview --- msgpack/_unpacker.pyx | 92 ++++++++++++++++++++++++++++++++++--------- msgpack/fallback.py | 37 ++++++++++------- test/test_buffer.py | 9 +++++ 3 files changed, 106 insertions(+), 32 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 23f6478..f6e06b0 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -8,16 +8,23 @@ from cpython.bytes cimport ( ) from cpython.buffer cimport ( Py_buffer, - PyBuffer_Release, + PyObject_CheckBuffer, PyObject_GetBuffer, + PyBuffer_Release, + PyBuffer_IsContiguous, + PyBUF_READ, PyBUF_SIMPLE, + PyBUF_FULL_RO, ) from cpython.mem cimport PyMem_Malloc, PyMem_Free from cpython.object cimport PyCallable_Check +from cpython.ref cimport Py_DECREF +from cpython.exc cimport PyErr_WarnEx cdef extern from "Python.h": ctypedef struct PyObject cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1 + object PyMemoryView_GetContiguous(object obj, int buffertype, char order) from libc.stdlib cimport * from libc.string cimport * @@ -110,6 +117,42 @@ cdef inline init_ctx(unpack_context *ctx, def default_read_extended_type(typecode, data): raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode) +cdef inline int get_data_from_buffer(object obj, + Py_buffer *view, + char **buf, + Py_ssize_t *buffer_len, + int *new_protocol) except 0: + cdef object contiguous + cdef Py_buffer tmp + if PyObject_CheckBuffer(obj): + new_protocol[0] = 1 + if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1: + raise + if view.itemsize != 1: + PyBuffer_Release(view) + raise BufferError("cannot unpack from multi-byte object") + if PyBuffer_IsContiguous(view, 'A') == 0: + PyBuffer_Release(view) + # create a contiguous copy and get buffer + contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, 'C') + PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE) + # view must hold the only reference to contiguous, + # so memory is freed when view is released + Py_DECREF(contiguous) + buffer_len[0] = view.len + buf[0] = view.buf + return 1 + else: + new_protocol[0] = 0 + if PyObject_AsReadBuffer(obj, buf, buffer_len) == -1: + raise BufferError("could not get memoryview") + PyErr_WarnEx(RuntimeWarning, + "using old buffer interface to unpack %s; " + "this leads to unpacking errors if slicing is used and " + "will be removed in a future version" % type(obj), + 1) + return 1 + def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=1, encoding=None, unicode_errors="strict", object_pairs_hook=None, ext_hook=ExtType, @@ -129,27 +172,34 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef Py_ssize_t off = 0 cdef int ret - cdef char* buf + cdef Py_buffer view + cdef char* buf = NULL cdef Py_ssize_t buf_len cdef char* cenc = NULL cdef char* cerr = NULL + cdef int new_protocol = 0 - PyObject_AsReadBuffer(packed, &buf, &buf_len) + get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol) - if encoding is not None: - if isinstance(encoding, unicode): - encoding = encoding.encode('ascii') - cenc = PyBytes_AsString(encoding) + try: + if encoding is not None: + if isinstance(encoding, unicode): + encoding = encoding.encode('ascii') + cenc = PyBytes_AsString(encoding) - if unicode_errors is not None: - if isinstance(unicode_errors, unicode): - unicode_errors = unicode_errors.encode('ascii') - cerr = PyBytes_AsString(unicode_errors) + if unicode_errors is not None: + if isinstance(unicode_errors, unicode): + unicode_errors = unicode_errors.encode('ascii') + cerr = PyBytes_AsString(unicode_errors) + + init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook, + use_list, cenc, cerr, + max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) + ret = unpack_construct(&ctx, buf, buf_len, &off) + finally: + if new_protocol: + PyBuffer_Release(&view); - init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook, - use_list, cenc, cerr, - max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) - ret = unpack_construct(&ctx, buf, buf_len, &off) if ret == 1: obj = unpack_data(&ctx) if off < buf_len: @@ -335,14 +385,20 @@ cdef class Unpacker(object): def feed(self, object next_bytes): """Append `next_bytes` to internal buffer.""" cdef Py_buffer pybuff + cdef int new_protocol = 0 + cdef char* buf + cdef Py_ssize_t buf_len + if self.file_like is not None: raise AssertionError( "unpacker.feed() is not be able to use with `file_like`.") - PyObject_GetBuffer(next_bytes, &pybuff, PyBUF_SIMPLE) + + get_data_from_buffer(next_bytes, &pybuff, &buf, &buf_len, &new_protocol) try: - self.append_buffer(pybuff.buf, pybuff.len) + self.append_buffer(buf, buf_len) finally: - PyBuffer_Release(&pybuff) + if new_protocol: + PyBuffer_Release(&pybuff) cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len): cdef: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index a23ad8c..11087eb 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -1,8 +1,8 @@ """Fallback pure Python implementation of msgpack""" import sys -import array import struct +import warnings if sys.version_info[0] == 3: PY3 = True @@ -46,6 +46,7 @@ else: from io import BytesIO as StringIO newlist_hint = lambda size: [] + from msgpack.exceptions import ( BufferFull, OutOfData, @@ -79,6 +80,24 @@ def _check_type_strict(obj, t, type=type, tuple=tuple): return type(obj) is t +def _get_data_from_buffer(obj): + try: + view = memoryview(obj) + except TypeError: + # try to use legacy buffer protocol if 2.7, otherwise re-raise + if not PY3: + view = memoryview(buffer(obj)) + warnings.warn("using old buffer interface to unpack %s; " + "this leads to unpacking errors if slicing is used and " + "will be removed in a future version" % type(obj), + RuntimeWarning) + else: + raise + if view.itemsize != 1: + raise ValueError("cannot unpack from multi-byte object") + return view + + def unpack(stream, **kwargs): """ Unpack an object from `stream`. @@ -239,17 +258,11 @@ class Unpacker(object): raise TypeError("`ext_hook` is not callable") def feed(self, next_bytes): - if isinstance(next_bytes, array.array): - next_bytes = next_bytes.tostring() - if not isinstance(next_bytes, (bytes, bytearray)): - raise TypeError("next_bytes should be bytes, bytearray or array.array") assert self._feeding - - if (len(self._buffer) - self._buff_i + len(next_bytes) > self._max_buffer_size): + view = _get_data_from_buffer(next_bytes) + if (len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size): raise BufferFull - # bytes + bytearray -> bytearray - # So cast before append - self._buffer += next_bytes + self._buffer += view def _consume(self): """ Gets rid of the used parts of the buffer. """ @@ -308,7 +321,6 @@ class Unpacker(object): n = 0 obj = None self._reserve(1) - #b = struct.unpack_from("B", self._buffer, self._buff_i)[0] b = self._buffer[self._buff_i] self._buff_i += 1 if b & 0b10000000 == 0: @@ -340,7 +352,6 @@ class Unpacker(object): elif b == 0xc4: typ = TYPE_BIN self._reserve(1) - #n = struct.unpack_from("B", self._buffer, self._buff_i)[0] n = self._buffer[self._buff_i] self._buff_i += 1 if n > self._max_bin_len: @@ -396,7 +407,6 @@ class Unpacker(object): self._buff_i += 8 elif b == 0xcc: self._reserve(1) - #obj = struct.unpack_from("B", self._buffer, self._buff_i)[0] obj = self._buffer[self._buff_i] self._buff_i += 1 elif b == 0xcd: @@ -465,7 +475,6 @@ class Unpacker(object): elif b == 0xd9: typ = TYPE_RAW self._reserve(1) - #n, = struct.unpack_from("B", self._buffer, self._buff_i) n = self._buffer[self._buff_i] self._buff_i += 1 if n > self._max_str_len: diff --git a/test/test_buffer.py b/test/test_buffer.py index 5a71f90..87f359f 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -18,3 +18,12 @@ def test_unpack_bytearray(): assert [b'foo', b'bar'] == obj expected_type = bytes assert all(type(s) == expected_type for s in obj) + + +def test_unpack_memoryview(): + buf = bytearray(packb(('foo', 'bar'))) + view = memoryview(buf) + obj = unpackb(view, use_list=1) + assert [b'foo', b'bar'] == obj + expected_type = bytes + assert all(type(s) == expected_type for s in obj) From d6254abc8a3ebec6a135b923951767bd97557de4 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 3 May 2016 11:58:28 +0900 Subject: [PATCH 0885/1172] Use AppVeyor to build windows wheel (#188) * Add AppVeyor support to build windows wheel * Fix test_limits on 32bit environments * Ignore Python35-x64 test fail for now Should be fixed in next version. --- appveyor.yml | 57 +++++++++++++++++++++++++++++++++++++++++++++ build.cmd | 21 +++++++++++++++++ msgpack/_packer.pyx | 4 ++-- 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 appveyor.yml create mode 100644 build.cmd diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..02b4461 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,57 @@ +environment: + + matrix: + + # For Python versions available on Appveyor, see + # http://www.appveyor.com/docs/installed-software#python + # The list here is complete (excluding Python 2.6, which + # isn't covered by this document) at the time of writing. + + - PYTHON: "C:\\Python27" + - PYTHON: "C:\\Python34" + - PYTHON: "C:\\Python35" + - PYTHON: "C:\\Python27-x64" + - PYTHON: "C:\\Python34-x64" + DISTUTILS_USE_SDK: "1" + + # Python35-x64 test fails with MemoryError + # TODO: investigate it + #- PYTHON: "C:\\Python35-x64" + +install: + # We need wheel installed to build wheels + - "%PYTHON%\\python.exe -m pip install -U pip wheel pytest cython" + +build: off + +test_script: + # Put your test command here. + # If you don't need to build C extensions on 64-bit Python 3.3 or 3.4, + # you can remove "build.cmd" from the front of the command, as it's + # only needed to support those cases. + # Note that you must use the environment variable %PYTHON% to refer to + # the interpreter you're using - Appveyor does not do anything special + # to put the Python evrsion you want to use on PATH. + - "build.cmd %PYTHON%\\python.exe setup.py build_ext -i" + - "build.cmd %PYTHON%\\python.exe setup.py install" + - "%PYTHON%\\python.exe -c \"import sys; print(hex(sys.maxsize))\"" + - "%PYTHON%\\python.exe -c \"from msgpack import _packer, _unpacker\"" + - "%PYTHON%\\Scripts\\py.test test" + - "build.cmd %PYTHON%\\python.exe setup.py bdist_wheel" + +after_test: + # This step builds your wheels. + # Again, you only need build.cmd if you're building C extensions for + # 64-bit Python 3.3/3.4. And you need to use %PYTHON% to get the correct + # interpreter + +artifacts: + # bdist_wheel puts your built wheel in the dist directory + - path: dist\* + +#on_success: +# You can use this step to upload your artifacts to a public website. +# See Appveyor's documentation for more details. Or you can simply +# access your wheels from the Appveyor "artifacts" tab for your build. + +# vim: set shiftwidth=2 diff --git a/build.cmd b/build.cmd new file mode 100644 index 0000000..243dc9a --- /dev/null +++ b/build.cmd @@ -0,0 +1,21 @@ +@echo off +:: To build extensions for 64 bit Python 3, we need to configure environment +:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 4 +:: +:: More details at: +:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows + +IF "%DISTUTILS_USE_SDK%"=="1" ( + ECHO Configuring environment to build with MSVC on a 64bit architecture + ECHO Using Windows SDK 7.1 + "C:\Program Files\Microsoft SDKs\Windows\v7.1\Setup\WindowsSdkVer.exe" -q -version:v7.1 + CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 /release + SET MSSdk=1 + REM Need the following to allow tox to see the SDK compiler + SET TOX_TESTENV_PASSENV=DISTUTILS_USE_SDK MSSdk INCLUDE LIB +) ELSE ( + ECHO Using default MSVC build environment +) + +CALL %* diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 6392655..872465b 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -244,7 +244,7 @@ cdef class Packer(object): msgpack_pack_ext(&self.pk, typecode, len(data)) msgpack_pack_raw_body(&self.pk, data, len(data)) - def pack_array_header(self, size_t size): + def pack_array_header(self, long long size): if size > (2**32-1): raise ValueError cdef int ret = msgpack_pack_array(&self.pk, size) @@ -257,7 +257,7 @@ cdef class Packer(object): self.pk.length = 0 return buf - def pack_map_header(self, size_t size): + def pack_map_header(self, long long size): if size > (2**32-1): raise ValueError cdef int ret = msgpack_pack_map(&self.pk, size) From 334dbe2a9652f43abdf27d978d9f4cdaf3f2a34d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 21 Jul 2016 19:19:32 +0900 Subject: [PATCH 0886/1172] Enable Python35-x64 in AppVeyor --- appveyor.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 02b4461..9e766c5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -13,10 +13,7 @@ environment: - PYTHON: "C:\\Python27-x64" - PYTHON: "C:\\Python34-x64" DISTUTILS_USE_SDK: "1" - - # Python35-x64 test fails with MemoryError - # TODO: investigate it - #- PYTHON: "C:\\Python35-x64" + - PYTHON: "C:\\Python35-x64" install: # We need wheel installed to build wheels From b911b3c652e190e6942a610fb3389aaaf2ccf3cc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 21 Jul 2016 19:32:00 +0900 Subject: [PATCH 0887/1172] Fix ext_hook call (#203) fixes #202 --- msgpack/unpack.h | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 92f4f11..da2cfb6 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -265,9 +265,9 @@ static inline int unpack_callback_ext(unpack_user* u, const char* base, const ch } // length also includes the typecode, so the actual data is length-1 #if PY_MAJOR_VERSION == 2 - py = PyObject_CallFunction(u->ext_hook, "(is#)", typecode, pos, length-1); + py = PyObject_CallFunction(u->ext_hook, "(is#)", (int)typecode, pos, (Py_ssize_t)length-1); #else - py = PyObject_CallFunction(u->ext_hook, "(iy#)", typecode, pos, length-1); + py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1); #endif if (!py) return -1; diff --git a/setup.py b/setup.py index 37729bd..1363586 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ except ImportError: def cythonize(src): sys.stderr.write("cythonize: %r\n" % (src,)) - cython_compiler.compile([src], cplus=True, emit_linenums=True) + cython_compiler.compile([src], cplus=True) def ensure_source(src): pyx = os.path.splitext(src)[0] + '.pyx' From ff208ad7d0b288f2d94d3160bf8a4cddeebc987d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 29 Jul 2016 22:25:05 +0900 Subject: [PATCH 0888/1172] 0.4.8 --- ChangeLog.rst | 11 ++++++++++- msgpack/_version.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 35535b4..2151736 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,6 +1,15 @@ +0.4.8 +===== +:release date: 2016-07-29 + +Bugs fixed +---------- + +* Calling ext_hook with wrong length. (Only on Windows, maybe. #203) + 0.4.7 ===== -:release date: TBD +:release date: 2016-01-25 Bugs fixed ---------- diff --git a/msgpack/_version.py b/msgpack/_version.py index 37c172d..76bd8fb 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 4, 7) +version = (0, 4, 8) From a9f4dad4dcde4db148f22720c694e5b5e0cb6f2d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 30 Jul 2016 11:35:26 +0900 Subject: [PATCH 0889/1172] Make manylinux1 wheels --- Makefile | 4 ++++ docker/buildwheel.sh | 11 +++++++++++ 2 files changed, 15 insertions(+) create mode 100644 docker/buildwheel.sh diff --git a/Makefile b/Makefile index 2e53d08..6eaef53 100644 --- a/Makefile +++ b/Makefile @@ -20,3 +20,7 @@ python3: cython test: py.test test + +build-manylinux1-wheel: + docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_i686 bash docker/buildwheel.sh + docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_x86_64 bash docker/buildwheel.sh diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh new file mode 100644 index 0000000..b654e45 --- /dev/null +++ b/docker/buildwheel.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e -x + +ARCH=`uname -p` +echo "arch=$ARCH" + +for V in cp35-cp35m cp34-cp34m cp27-cp27m cp27-cp27mu; do + PYBIN=/opt/python/$V/bin + rm -rf build/ # Avoid lib build by narrow Python is used by wide python + $PYBIN/python setup.py bdist_wheel -p manylinux1_${ARCH} +done From e3fea94509767047a8ff45aa07cd58a9ba9694e7 Mon Sep 17 00:00:00 2001 From: TW Date: Wed, 11 Jan 2017 04:04:23 +0100 Subject: [PATCH 0890/1172] fix typos and other cosmetic issues (#214) cosmetic issues: - reST headlines' underline length needs to match the headline length (looks like somebody is / was using a proportional font) - Cython code lines do not need to be terminated with a semicolon - always use triple-double-quotes for docstrings --- ChangeLog.rst | 46 +++++++++++++++++++++--------------------- README.rst | 29 +++++++++++++------------- appveyor.yml | 2 +- docs/api.rst | 2 +- docs/index.rst | 2 +- msgpack/_packer.pyx | 10 ++++----- msgpack/_unpacker.pyx | 2 +- msgpack/fallback.py | 16 +++++++-------- test/test_extension.py | 2 +- test/test_pack.py | 2 +- 10 files changed, 57 insertions(+), 56 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 2151736..30e6c5f 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -160,7 +160,7 @@ Changes 0.2.4 -======= +===== :release date: 2012-12-22 Bugs fixed @@ -169,7 +169,7 @@ Bugs fixed * Fix SEGV when object_hook or object_pairs_hook raise Exception. (#39) 0.2.3 -======= +===== :release date: 2012-12-11 Changes @@ -177,11 +177,11 @@ Changes * Warn when use_list is not specified. It's default value will be changed in 0.3. Bugs fixed ------------ +---------- * Can't pack subclass of dict. 0.2.2 -======= +===== :release date: 2012-09-21 Changes @@ -190,7 +190,7 @@ Changes object in single precision format. Bugs fixed ------------ +---------- * ``unpack()`` didn't restores gc state when it called with gc disabled. ``unpack()`` doesn't control gc now instead of restoring gc state collectly. User can control gc state when gc cause performance issue. @@ -198,7 +198,7 @@ Bugs fixed * ``Unpacker``'s ``read_size`` option didn't used. 0.2.1 -======= +===== :release date: 2012-08-20 Changes @@ -206,8 +206,8 @@ Changes * Add ``max_buffer_size`` parameter to Unpacker. It limits internal buffer size and allows unpack data from untrusted source safely. -* Unpacker's buffer reallocation algorithm is less greedy now. It cause perforamce - derease in rare case but memory efficient and don't allocate than ``max_buffer_size``. +* Unpacker's buffer reallocation algorithm is less greedy now. It cause performance + decrease in rare case but memory efficient and don't allocate than ``max_buffer_size``. Bugs fixed ---------- @@ -217,7 +217,7 @@ Bugs fixed 0.2.0 -======= +===== :release date: 2012-06-27 Changes @@ -232,16 +232,16 @@ Bugs fixed 0.1.13 -======= +====== :release date: 2012-04-21 New ----- +--- * Don't accept subtype of list and tuple as msgpack list. (Steeve Morin) It allows customize how it serialized with ``default`` argument. Bugs fixed ------------ +---------- * Fix wrong error message. (David Wolever) * Fix memory leak while unpacking when ``object_hook`` or ``list_hook`` is used. (Steeve Morin) @@ -253,21 +253,21 @@ Other changes 0.1.12 -======= +====== :release date: 2011-12-27 Bugs fixed -------------- +---------- * Re-enable packs/unpacks removed at 0.1.11. It will be removed when 0.2 is released. 0.1.11 -======= +====== :release date: 2011-12-26 Bugs fixed -------------- +---------- * Include test code for Python3 to sdist. (Johan Bergström) * Fix compilation error on MSVC. (davidgaleano) @@ -285,7 +285,7 @@ New feature 0.1.9 -====== +===== :release date: 2011-01-29 New feature @@ -299,16 +299,16 @@ Bugs fixed * Add MemoryError check. 0.1.8 -====== +===== :release date: 2011-01-10 New feature ------------- +----------- * Support ``loads`` and ``dumps`` aliases for API compatibility with simplejson and pickle. * Add *object_hook* and *list_hook* option to unpacker. It allows you to - hook unpacing mapping type and array type. + hook unpacking mapping type and array type. * Add *default* option to packer. It allows you to pack unsupported types. @@ -320,13 +320,13 @@ Bugs fixed 0.1.7 -====== +===== :release date: 2010-11-02 New feature ------------- +----------- * Add *object_hook* and *list_hook* option to unpacker. It allows you to - hook unpacing mapping type and array type. + hook unpacking mapping type and array type. * Add *default* option to packer. It allows you to pack unsupported types. diff --git a/README.rst b/README.rst index d32ec1d..e37c61d 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ -======================= +====================== MessagePack for Python -======================= +====================== :author: INADA Naoki :version: 0.4.6 @@ -11,21 +11,22 @@ MessagePack for Python What's this ------------- +----------- -`MessagePack `_ is a fast, compact binary serialization format, suitable for -similar data to JSON. This package provides CPython bindings for reading and -writing MessagePack data. +`MessagePack `_ is an efficient binary serialization format. +It lets you exchange data among multiple languages like JSON. +But it's faster and smaller. +This package provides CPython bindings for reading and writing MessagePack data. Install ---------- +------- :: $ pip install msgpack-python PyPy -^^^^^ +^^^^ msgpack-python provides pure python implementation. PyPy can use this. @@ -44,7 +45,7 @@ Community Edition or Express Edition can be used to build extension module. How to use ------------ +---------- One-shot pack & unpack ^^^^^^^^^^^^^^^^^^^^^^ @@ -134,7 +135,7 @@ It is also possible to pack/unpack custom data types. Here is an example for key-value pairs. Extended types -^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^ It is also possible to pack/unpack custom data types using the **ext** type. @@ -166,7 +167,7 @@ Advanced unpacking control As an alternative to iteration, ``Unpacker`` objects provide ``unpack``, ``skip``, ``read_array_header`` and ``read_map_header`` methods. The former two -read an entire message from the stream, respectively deserialising and returning +read an entire message from the stream, respectively de-serialising and returning the result, or ignoring it. The latter two methods return the number of elements in the upcoming container, so that each element in an array, or key-value pair in a map, can be unpacked or skipped individually. @@ -243,7 +244,7 @@ instead of `StopIteration`. `StopIteration` is used for iterator protocol only. Note about performance ------------------------- +---------------------- GC ^^ @@ -253,7 +254,7 @@ This means unpacking may cause useless GC. You can use ``gc.disable()`` when unpacking large message. use_list option -^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^ List is the default sequence type of Python. But tuple is lighter than list. You can use ``use_list=False`` while unpacking when performance is important. @@ -264,7 +265,7 @@ Another way to unpacking such object is using ``object_pairs_hook``. Development ------------- +----------- Test ^^^^ diff --git a/appveyor.yml b/appveyor.yml index 9e766c5..a8a2352 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,7 +28,7 @@ test_script: # only needed to support those cases. # Note that you must use the environment variable %PYTHON% to refer to # the interpreter you're using - Appveyor does not do anything special - # to put the Python evrsion you want to use on PATH. + # to put the Python version you want to use on PATH. - "build.cmd %PYTHON%\\python.exe setup.py build_ext -i" - "build.cmd %PYTHON%\\python.exe setup.py install" - "%PYTHON%\\python.exe -c \"import sys; print(hex(sys.maxsize))\"" diff --git a/docs/api.rst b/docs/api.rst index 841c134..6336793 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -28,7 +28,7 @@ API reference .. autoclass:: ExtType exceptions ------------ +---------- These exceptions are accessible via `msgpack` package. (For example, `msgpack.OutOfData` is shortcut for `msgpack.exceptions.OutOfData`) diff --git a/docs/index.rst b/docs/index.rst index 72d4499..dcdab4f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,5 +1,5 @@ msgpack document -================== +================ `MessagePack `_ is a efficient format for inter language data exchange. diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 872465b..7a12853 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -58,11 +58,11 @@ cdef class Packer(object): :param bool use_single_float: Use single precision float type for float. (default: False) :param bool autoreset: - Reset buffer after each pack and return it's content as `bytes`. (default: True). + Reset buffer after each pack and return its content as `bytes`. (default: True). If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. :param bool use_bin_type: Use bin type introduced in msgpack spec 2.0 for bytes. - It also enable str8 type for unicode. + It also enables str8 type for unicode. """ cdef msgpack_packer pk cdef object _default @@ -75,7 +75,7 @@ cdef class Packer(object): def __cinit__(self): cdef int buf_size = 1024*1024 - self.pk.buf = malloc(buf_size); + self.pk.buf = malloc(buf_size) if self.pk.buf == NULL: raise MemoryError("Unable to allocate internal buffer.") self.pk.buf_size = buf_size @@ -108,7 +108,7 @@ cdef class Packer(object): self.unicode_errors = PyBytes_AsString(self._berrors) def __dealloc__(self): - free(self.pk.buf); + free(self.pk.buf) cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: cdef long long llval @@ -274,7 +274,7 @@ cdef class Packer(object): """ Pack *pairs* as msgpack map type. - *pairs* should sequence of pair. + *pairs* should be a sequence of pairs. (`len(pairs)` and `for k, v in pairs:` should be supported.) """ cdef int ret = msgpack_pack_map(&self.pk, len(pairs)) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 1aefc64..2a13903 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -209,7 +209,7 @@ cdef class Unpacker(object): :param int max_buffer_size: Limits size of data waiting unpacked. 0 means system's INT_MAX (default). Raises `BufferFull` exception when it is insufficient. - You shoud set this parameter when unpacking data from untrusted source. + You should set this parameter when unpacking data from untrusted source. :param int max_str_len: Limits max length of str. (default: 2**31-1) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index f682611..fefabb8 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -138,7 +138,7 @@ class Unpacker(object): :param int max_buffer_size: Limits size of data waiting unpacked. 0 means system's INT_MAX (default). Raises `BufferFull` exception when it is insufficient. - You shoud set this parameter when unpacking data from untrusted source. + You should set this parameter when unpacking data from untrusted source. :param int max_str_len: Limits max length of str. (default: 2**31-1) @@ -188,13 +188,13 @@ class Unpacker(object): self.file_like = file_like self._fb_feeding = False - #: array of bytes feeded. + #: array of bytes fed. self._fb_buffers = [] #: Which buffer we currently reads self._fb_buf_i = 0 #: Which position we currently reads self._fb_buf_o = 0 - #: Total size of _fb_bufferes + #: Total size of _fb_buffers self._fb_buf_n = 0 # When Unpacker is used as an iterable, between the calls to next(), @@ -203,7 +203,7 @@ class Unpacker(object): # the correct moments, we have to keep track of how sloppy we were. # Furthermore, when the buffer is incomplete (that is: in the case # we raise an OutOfData) we need to rollback the buffer to the correct - # state, which _fb_slopiness records. + # state, which _fb_sloppiness records. self._fb_sloppiness = 0 self._max_buffer_size = max_buffer_size or 2**31-1 @@ -303,7 +303,7 @@ class Unpacker(object): def _fb_read(self, n, write_bytes=None): buffs = self._fb_buffers - # We have a redundant codepath for the most common case, such that + # We have a redundant code path for the most common case, such that # pypy optimizes it properly. This is the case that the read fits # in the current buffer. if (write_bytes is None and self._fb_buf_i < len(buffs) and @@ -598,17 +598,17 @@ class Packer(object): Convert user type to builtin type that Packer supports. See also simplejson's document. :param str encoding: - Convert unicode to bytes with this encoding. (default: 'utf-8') + Convert unicode to bytes with this encoding. (default: 'utf-8') :param str unicode_errors: Error handler for encoding unicode. (default: 'strict') :param bool use_single_float: Use single precision float type for float. (default: False) :param bool autoreset: - Reset buffer after each pack and return it's content as `bytes`. (default: True). + Reset buffer after each pack and return its content as `bytes`. (default: True). If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. :param bool use_bin_type: Use bin type introduced in msgpack spec 2.0 for bytes. - It also enable str8 type for unicode. + It also enables str8 type for unicode. """ def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, autoreset=True, use_bin_type=False): diff --git a/test/test_extension.py b/test/test_extension.py index c552498..d05d7ab 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -42,7 +42,7 @@ def test_extension_type(): typecode = 123 # application specific typecode data = obj.tostring() return ExtType(typecode, data) - raise TypeError("Unknwon type object %r" % (obj,)) + raise TypeError("Unknown type object %r" % (obj,)) def ext_hook(code, data): print('ext_hook called', code, data) diff --git a/test/test_pack.py b/test/test_pack.py index 762ccf5..e945902 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -130,7 +130,7 @@ def testMapSize(sizes=[0, 5, 50, 1000]): class odict(dict): - '''Reimplement OrderedDict to run test on Python 2.6''' + """Reimplement OrderedDict to run test on Python 2.6""" def __init__(self, seq): self._seq = seq dict.__init__(self, seq) From f985ee8a665daa298282ab87c488e81b7d9814a7 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 13 Jan 2017 19:57:04 +0900 Subject: [PATCH 0891/1172] Remove version and date from README --- README.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.rst b/README.rst index 74f5fc7..da625b4 100644 --- a/README.rst +++ b/README.rst @@ -3,8 +3,6 @@ MessagePack for Python ====================== :author: INADA Naoki -:version: 0.4.6 -:date: 2015-03-13 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.svg :target: https://travis-ci.org/#!/msgpack/msgpack-python From 12845692b5a3703bc88e85f492143ca1a9985902 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 13 Jan 2017 20:41:33 +0900 Subject: [PATCH 0892/1172] Add requirements.txt for Read the Docs --- requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..87f04da --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +Cython==0.25.2 From b328f3ecffd22e7f0db76e81774727fd171bf303 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 13 Jan 2017 20:48:48 +0900 Subject: [PATCH 0893/1172] Add badge for Read the Docs --- README.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index da625b4..0e1ddd7 100644 --- a/README.rst +++ b/README.rst @@ -2,11 +2,12 @@ MessagePack for Python ====================== -:author: INADA Naoki - .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.svg :target: https://travis-ci.org/#!/msgpack/msgpack-python - + +.. image:: https://readthedocs.org/projects/msgpack-python/badge/?version=latest + :target: http://msgpack-python.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status What's this ----------- From 3388e4a6ee6adea56789d97cc05ed610a4e5b4fc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 13 Jan 2017 21:46:31 +0900 Subject: [PATCH 0894/1172] travis and appveyor update (#217) travis: * stop using tox * Add Python 3.6 and 3.7-dev * Stop pypy3 (until PyPy3.5 is released) appveyor: * Drop Python 3.4 and add 3.6 --- .travis.yml | 34 +++++++++++++++++++++++----------- appveyor.yml | 8 ++++---- docker/runtests.sh | 2 +- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index b4396cb..0170360 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,22 +1,23 @@ sudo: false language: python -python: 3.5 -cache: - directories: - - $HOME/.cache/pip +cache: pip + +python: + - "2.7" + - "3.3" + - "3.4" + - "3.5" + - "3.6" + - "3.7-dev" branches: only: - master -env: - - TOXENV=py27-c,py33-c,py34-c,py35-c - - TOXENV=py27-pure,py33-pure,py34-pure,py35-pure - - TOXENV=pypy-pure,pypy3-pure - matrix: include: - sudo: required + language: c services: - docker env: @@ -28,12 +29,23 @@ matrix: - docker pull $DOCKER_IMAGE script: - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh + - python: "pypy" + install: + - pip install -e . + script: + - py.test -v test + install: - pip install -U pip - - pip install tox cython + - pip install cython - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx + - pip install -e . -script: tox +script: + - python -c 'import sys; print(hex(sys.maxsize))' + - python -c 'from msgpack import _packer, _unpacker' + - py.test -v test + - MSGPACK_PUREPYTHON=x py.test -v test # vim: sw=2 ts=2 diff --git a/appveyor.yml b/appveyor.yml index a8a2352..e63423d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -8,16 +8,16 @@ environment: # isn't covered by this document) at the time of writing. - PYTHON: "C:\\Python27" - - PYTHON: "C:\\Python34" - - PYTHON: "C:\\Python35" - PYTHON: "C:\\Python27-x64" - - PYTHON: "C:\\Python34-x64" - DISTUTILS_USE_SDK: "1" + - PYTHON: "C:\\Python35" - PYTHON: "C:\\Python35-x64" + - PYTHON: "C:\\Python36" + - PYTHON: "C:\\Python36-x64" install: # We need wheel installed to build wheels - "%PYTHON%\\python.exe -m pip install -U pip wheel pytest cython" + - "%PYTHON%\\Scripts\\cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx" build: off diff --git a/docker/runtests.sh b/docker/runtests.sh index 0d74802..0eea715 100755 --- a/docker/runtests.sh +++ b/docker/runtests.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -x -for V in cp35-cp35m cp34-cp34m cp27-cp27m cp27-cp27mu; do +for V in cp36-cp36m cp35-cp35m cp34-cp34m cp27-cp27m cp27-cp27mu; do PYBIN=/opt/python/$V/bin $PYBIN/python setup.py install rm -rf build/ # Avoid lib build by narrow Python is used by wide python From a8d9162ca6cff6101c1f6b9547e94749c6acae96 Mon Sep 17 00:00:00 2001 From: jfolz Date: Sat, 29 Apr 2017 19:33:20 +0200 Subject: [PATCH 0895/1172] Unpacker: add tell() (#227) --- msgpack/_unpacker.pyx | 7 +++++++ msgpack/fallback.py | 5 +++++ test/test_sequnpack.py | 20 ++++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index a9801ee..dabc5f7 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -29,6 +29,7 @@ cdef extern from "Python.h": from libc.stdlib cimport * from libc.string cimport * from libc.limits cimport * +ctypedef unsigned long long uint64_t from msgpack.exceptions import ( BufferFull, @@ -314,6 +315,7 @@ cdef class Unpacker(object): cdef object object_hook, object_pairs_hook, list_hook, ext_hook cdef object encoding, unicode_errors cdef Py_ssize_t max_buffer_size + cdef uint64_t stream_offset def __cinit__(self): self.buf = NULL @@ -358,6 +360,7 @@ cdef class Unpacker(object): self.buf_size = read_size self.buf_head = 0 self.buf_tail = 0 + self.stream_offset = 0 if encoding is not None: if isinstance(encoding, unicode): @@ -468,6 +471,7 @@ cdef class Unpacker(object): try: ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + self.stream_offset += self.buf_head - prev_head if write_bytes is not None: write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head)) @@ -534,6 +538,9 @@ cdef class Unpacker(object): """ return self._unpack(read_map_header, write_bytes) + def tell(self): + return self.stream_offset + def __iter__(self): return self diff --git a/msgpack/fallback.py b/msgpack/fallback.py index d2eb9f4..508fd06 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -244,6 +244,7 @@ class Unpacker(object): self._max_array_len = max_array_len self._max_map_len = max_map_len self._max_ext_len = max_ext_len + self._stream_offset = 0 if list_hook is not None and not callable(list_hook): raise TypeError('`list_hook` is not callable') @@ -266,6 +267,7 @@ class Unpacker(object): def _consume(self): """ Gets rid of the used parts of the buffer. """ + self._stream_offset += self._buff_i - self._buf_checkpoint self._buf_checkpoint = self._buff_i def _got_extradata(self): @@ -629,6 +631,9 @@ class Unpacker(object): self._consume() return ret + def tell(self): + return self._stream_offset + class Packer(object): """ diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 45f4cc7..59718f5 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -3,6 +3,7 @@ import io from msgpack import Unpacker, BufferFull +from msgpack import pack from msgpack.exceptions import OutOfData from pytest import raises @@ -96,3 +97,22 @@ def test_issue124(): unpacker.feed(b"!") assert tuple(unpacker) == (b'!',) assert tuple(unpacker) == () + + +def test_unpack_tell(): + stream = io.BytesIO() + messages = [2**i-1 for i in range(65)] + messages += [-(2**i) for i in range(1, 64)] + messages += [b'hello', b'hello'*1000, list(range(20)), + {i: bytes(i)*i for i in range(10)}, + {i: bytes(i)*i for i in range(32)}] + offsets = [] + for m in messages: + pack(m, stream) + offsets.append(stream.tell()) + stream.seek(0) + unpacker = Unpacker(stream) + for m, o in zip(messages, offsets): + m2 = next(unpacker) + assert m == m2 + assert o == unpacker.tell() From f0f2c0b39703e0129d2352c71ec9811a8f275cc8 Mon Sep 17 00:00:00 2001 From: jfolz Date: Thu, 18 May 2017 13:03:15 +0200 Subject: [PATCH 0896/1172] Packer accepts bytearray objects (#229) --- msgpack/_packer.pyx | 14 ++++++++++++-- msgpack/fallback.py | 6 ++++-- test/test_pack.py | 7 +++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index f24aa70..5a81709 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -10,6 +10,8 @@ from msgpack import ExtType cdef extern from "Python.h": int PyMemoryView_Check(object obj) + int PyByteArray_Check(object obj) + int PyByteArray_CheckExact(object obj) cdef extern from "pack.h": @@ -39,6 +41,14 @@ cdef int DEFAULT_RECURSE_LIMIT=511 cdef size_t ITEM_LIMIT = (2**32)-1 +cdef inline int PyBytesLike_Check(object o): + return PyBytes_Check(o) or PyByteArray_Check(o) + + +cdef inline int PyBytesLike_CheckExact(object o): + return PyBytes_CheckExact(o) or PyByteArray_CheckExact(o) + + cdef class Packer(object): """ MessagePack Packer @@ -174,10 +184,10 @@ cdef class Packer(object): else: dval = o ret = msgpack_pack_double(&self.pk, dval) - elif PyBytes_CheckExact(o) if strict_types else PyBytes_Check(o): + elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): L = len(o) if L > ITEM_LIMIT: - raise PackValueError("bytes is too large") + raise PackValueError("%s is too large" % type(o).__name__) rawval = o ret = msgpack_pack_bin(&self.pk, L) if ret == 0: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 508fd06..a02cbe1 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -38,6 +38,8 @@ if hasattr(sys, 'pypy_version_info'): def write(self, s): if isinstance(s, memoryview): s = s.tobytes() + elif isinstance(s, bytearray): + s = bytes(s) self.builder.append(s) def getvalue(self): return self.builder.build() @@ -728,10 +730,10 @@ class Packer(object): default_used = True continue raise PackOverflowError("Integer value out of range") - if check(obj, bytes): + if check(obj, (bytes, bytearray)): n = len(obj) if n >= 2**32: - raise PackValueError("Bytes is too large") + raise PackValueError("%s is too large" % type(obj).__name__) self._pack_bin_header(n) return self._buffer.write(obj) if check(obj, Unicode): diff --git a/test/test_pack.py b/test/test_pack.py index e945902..a704fdb 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -58,6 +58,13 @@ def testPackBytes(): for td in test_data: check(td) +def testPackByteArrays(): + test_data = [ + bytearray(b""), bytearray(b"abcd"), (bytearray(b"defgh"),), + ] + for td in test_data: + check(td) + def testIgnoreUnicodeErrors(): re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1) assert re == "abcdef" From deeda31a8840cee334f05f15bd2308af13dc9c64 Mon Sep 17 00:00:00 2001 From: Lorenzo Bolla Date: Sat, 30 Sep 2017 08:23:55 +0100 Subject: [PATCH 0897/1172] Add unittests to document serialisation of tuples (#246) Also, fix formatting of error message in case of tuple. See https://github.com/msgpack/msgpack-python/issues/245 --- msgpack/fallback.py | 2 +- test/test_stricttype.py | 49 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index a02cbe1..28478ca 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -795,7 +795,7 @@ class Packer(object): obj = self._default(obj) default_used = 1 continue - raise TypeError("Cannot serialize %r" % obj) + raise TypeError("Cannot serialize %r" % (obj, )) def pack(self, obj): self._pack(obj) diff --git a/test/test_stricttype.py b/test/test_stricttype.py index a20b5eb..0f865c8 100644 --- a/test/test_stricttype.py +++ b/test/test_stricttype.py @@ -1,7 +1,7 @@ # coding: utf-8 from collections import namedtuple -from msgpack import packb, unpackb +from msgpack import packb, unpackb, ExtType def test_namedtuple(): @@ -13,3 +13,50 @@ def test_namedtuple(): packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default) unpacked = unpackb(packed, encoding='utf-8') assert unpacked == {'foo': 1, 'bar': 42} + + +def test_tuple(): + t = ('one', 2, b'three', (4, )) + + def default(o): + if isinstance(o, tuple): + return { + '__type__': 'tuple', + 'value': list(o), + } + raise TypeError('Unsupported type %s' % (type(o),)) + + def convert(o): + if o.get('__type__') == 'tuple': + return tuple(o['value']) + return o + + data = packb(t, strict_types=True, use_bin_type=True, default=default) + expected = unpackb(data, encoding='utf-8', object_hook=convert) + + assert expected == t + + +def test_tuple_ext(): + t = ('one', 2, b'three', (4, )) + + MSGPACK_EXT_TYPE_TUPLE = 0 + + def default(o): + if isinstance(o, tuple): + # Convert to list and pack + payload = packb( + list(o), strict_types=True, use_bin_type=True, default=default) + return ExtType(MSGPACK_EXT_TYPE_TUPLE, payload) + raise TypeError(repr(o)) + + def convert(code, payload): + if code == MSGPACK_EXT_TYPE_TUPLE: + # Unpack and convert to tuple + return tuple(unpackb(payload, encoding='utf-8', ext_hook=convert)) + raise ValueError('Unknown Ext code {}'.format(code)) + + data = packb(t, strict_types=True, use_bin_type=True, default=default) + expected = unpackb(data, encoding='utf-8', ext_hook=convert) + + assert expected == t From b57106c246867b5beec62874a239c87d94dafba5 Mon Sep 17 00:00:00 2001 From: Hugo Date: Wed, 11 Oct 2017 20:49:02 +0300 Subject: [PATCH 0898/1172] Update badges (#247) --- README.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 0e1ddd7..ea1499b 100644 --- a/README.rst +++ b/README.rst @@ -2,17 +2,18 @@ MessagePack for Python ====================== -.. image:: https://secure.travis-ci.org/msgpack/msgpack-python.svg - :target: https://travis-ci.org/#!/msgpack/msgpack-python +.. image:: https://travis-ci.org/msgpack/msgpack-python.svg?branch=master + :target: https://travis-ci.org/msgpack/msgpack-python + :alt: Build Status .. image:: https://readthedocs.org/projects/msgpack-python/badge/?version=latest - :target: http://msgpack-python.readthedocs.io/en/latest/?badge=latest + :target: https://msgpack-python.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status What's this ----------- -`MessagePack `_ is an efficient binary serialization format. +`MessagePack `_ is an efficient binary serialization format. It lets you exchange data among multiple languages like JSON. But it's faster and smaller. This package provides CPython bindings for reading and writing MessagePack data. From 54aa47b2dd489297d894c0639811653fd6ff7bfa Mon Sep 17 00:00:00 2001 From: Hugo Date: Thu, 12 Oct 2017 03:26:34 +0300 Subject: [PATCH 0899/1172] Update supported versions in classifiers (#248) --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.py b/setup.py index 5f0f08c..5c88397 100755 --- a/setup.py +++ b/setup.py @@ -114,7 +114,13 @@ setup(name='msgpack-python', url='http://msgpack.org/', classifiers=[ 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', ] From 6fd1890be4692cca1a0c53cc160ac5da83e1d272 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 13 Jan 2017 21:47:26 +0900 Subject: [PATCH 0900/1172] Add py36 to tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b6e7a7f..b2ac336 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py33,py34,py35}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86 +envlist = {py27,py33,py34,py35,py36}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86 [variants:pure] setenv= From 3d7ebc47b4fed0cc06f652013c34f32107728c98 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 12 Oct 2017 15:28:40 +0900 Subject: [PATCH 0901/1172] travis: Remove "only: master" restriction --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0170360..f5141d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,10 +10,6 @@ python: - "3.6" - "3.7-dev" -branches: - only: - - master - matrix: include: - sudo: required From a70ce0c3d7bb25646302fd2649ba916178cf4a69 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 12 Oct 2017 16:26:58 +0900 Subject: [PATCH 0902/1172] Fix travis fail (#251) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f5141d6..54e0c62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ python: matrix: include: - sudo: required - language: c + language: python services: - docker env: From 0fc4ee98be498f39a320eff501ba30c49c31482d Mon Sep 17 00:00:00 2001 From: Hugo Date: Thu, 12 Oct 2017 10:27:39 +0300 Subject: [PATCH 0903/1172] Remove code and tests for unsupported Python 2.6 (#250) --- .gitignore | 1 + test/test_pack.py | 18 ++---------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 70f5746..800f1c2 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ msgpack/*.cpp /venv /tags /docs/_build +.cache diff --git a/test/test_pack.py b/test/test_pack.py index a704fdb..ac93103 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -7,6 +7,7 @@ from pytest import raises, xfail from msgpack import packb, unpackb, Unpacker, Packer +from collections import OrderedDict from io import BytesIO def check(data, use_list=False): @@ -136,24 +137,9 @@ def testMapSize(sizes=[0, 5, 50, 1000]): assert unpacker.unpack() == dict((i, i * 2) for i in range(size)) -class odict(dict): - """Reimplement OrderedDict to run test on Python 2.6""" - def __init__(self, seq): - self._seq = seq - dict.__init__(self, seq) - - def items(self): - return self._seq[:] - - def iteritems(self): - return iter(self._seq) - - def keys(self): - return [x[0] for x in self._seq] - def test_odict(): seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)] - od = odict(seq) + od = OrderedDict(seq) assert unpackb(packb(od), use_list=1) == dict(seq) def pair_hook(seq): return list(seq) From 1985eb7618296e6a93c8abc4a697c7b00fda72f8 Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Mon, 16 Oct 2017 20:30:55 -0700 Subject: [PATCH 0904/1172] Clarify README, fix grammar, update section on byte arrays (#253) --- README.rst | 59 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/README.rst b/README.rst index ea1499b..3c16078 100644 --- a/README.rst +++ b/README.rst @@ -28,14 +28,14 @@ Install PyPy ^^^^ -msgpack-python provides pure python implementation. PyPy can use this. +msgpack-python provides a pure Python implementation. PyPy can use this. Windows ^^^^^^^ -When you can't use binary distribution, you need to install Visual Studio +When you can't use a binary distribution, you need to install Visual Studio or Windows SDK on Windows. -Without extension, using pure python implementation on CPython runs slowly. +Without extension, using pure Python implementation on CPython runs slowly. For Python 2.7, `Microsoft Visual C++ Compiler for Python 2.7 `_ is recommended solution. @@ -51,11 +51,11 @@ One-shot pack & unpack ^^^^^^^^^^^^^^^^^^^^^^ Use ``packb`` for packing and ``unpackb`` for unpacking. -msgpack provides ``dumps`` and ``loads`` as alias for compatibility with +msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with ``json`` and ``pickle``. -``pack`` and ``dump`` packs to file-like object. -``unpack`` and ``load`` unpacks from file-like object. +``pack`` and ``dump`` packs to a file-like object. +``unpack`` and ``load`` unpacks from a file-like object. .. code-block:: pycon @@ -65,14 +65,15 @@ msgpack provides ``dumps`` and ``loads`` as alias for compatibility with >>> msgpack.unpackb(_) [1, 2, 3] -``unpack`` unpacks msgpack's array to Python's list, but can unpack to tuple: +``unpack`` unpacks msgpack's array to Python's list, but can also unpack to tuple: .. code-block:: pycon >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False) (1, 2, 3) -You should always pass the ``use_list`` keyword argument. See performance issues relating to `use_list option`_ below. +You should always specify the ``use_list`` keyword argument for backward compatibility. +See performance issues relating to `use_list option`_ below. Read the docstring for other options. @@ -198,29 +199,43 @@ Notes string and binary type ^^^^^^^^^^^^^^^^^^^^^^ -In old days, msgpack doesn't distinguish string and binary types like Python 1. -The type for represent string and binary types is named **raw**. +Early versions of msgpack didn't distinguish string and binary types (like Python 1). +The type for representing both string and binary types was named **raw**. -msgpack can distinguish string and binary type for now. But it is not like Python 2. -Python 2 added unicode string. But msgpack renamed **raw** to **str** and added **bin** type. -It is because keep compatibility with data created by old libs. **raw** was used for text more than binary. +For backward compatibility reasons, msgpack-python will still default all +strings to byte strings, unless you specify the `use_bin_type=True` option in +the packer. If you do so, it will use a non-standard type called **bin** to +serialize byte arrays, and **raw** becomes to mean **str**. If you want to +distinguish **bin** and **raw** in the unpacker, specify `encoding='utf-8'`. -Currently, while msgpack-python supports new **bin** type, default setting doesn't use it and -decodes **raw** as `bytes` instead of `unicode` (`str` in Python 3). - -You can change this by using `use_bin_type=True` option in Packer and `encoding="utf-8"` option in Unpacker. +Note that Python 2 defaults to byte-arrays over Unicode strings: .. code-block:: pycon >>> import msgpack - >>> packed = msgpack.packb([b'spam', u'egg'], use_bin_type=True) - >>> msgpack.unpackb(packed, encoding='utf-8') - ['spam', u'egg'] + >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'])) + ['spam', 'eggs'] + >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), + encoding='utf-8') + ['spam', u'eggs'] + +This is the same code in Python 3 (same behaviour, but Python 3 has a +different default): + +.. code-block:: pycon + + >>> import msgpack + >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'])) + [b'spam', b'eggs'] + >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), + encoding='utf-8') + [b'spam', 'eggs'] + ext type ^^^^^^^^ -To use **ext** type, pass ``msgpack.ExtType`` object to packer. +To use the **ext** type, pass ``msgpack.ExtType`` object to packer. .. code-block:: pycon @@ -234,7 +249,7 @@ You can use it with ``default`` and ``ext_hook``. See below. Note for msgpack-python 0.2.x users ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The msgpack-python 0.3 have some incompatible changes. +The msgpack-python release 0.3 has some incompatible changes. The default value of ``use_list`` keyword argument is ``True`` from 0.3. You should pass the argument explicitly for backward compatibility. From 3a098851bea500ef1ffde856a60d80ddab230dee Mon Sep 17 00:00:00 2001 From: Hugo Date: Thu, 2 Nov 2017 11:06:15 +0200 Subject: [PATCH 0905/1172] Remove code and tests for unsupported Python 3.3 and 3.4 (#249) --- .travis.yml | 2 -- appveyor.yml | 13 ++++--------- build.cmd | 21 --------------------- docker/runtests.sh | 2 +- tox.ini | 2 +- 5 files changed, 6 insertions(+), 34 deletions(-) delete mode 100644 build.cmd diff --git a/.travis.yml b/.travis.yml index 54e0c62..7aac664 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,6 @@ cache: pip python: - "2.7" - - "3.3" - - "3.4" - "3.5" - "3.6" - "3.7-dev" diff --git a/appveyor.yml b/appveyor.yml index e63423d..d581839 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,24 +23,19 @@ build: off test_script: # Put your test command here. - # If you don't need to build C extensions on 64-bit Python 3.3 or 3.4, - # you can remove "build.cmd" from the front of the command, as it's - # only needed to support those cases. # Note that you must use the environment variable %PYTHON% to refer to # the interpreter you're using - Appveyor does not do anything special # to put the Python version you want to use on PATH. - - "build.cmd %PYTHON%\\python.exe setup.py build_ext -i" - - "build.cmd %PYTHON%\\python.exe setup.py install" + - "%PYTHON%\\python.exe setup.py build_ext -i" + - "%PYTHON%\\python.exe setup.py install" - "%PYTHON%\\python.exe -c \"import sys; print(hex(sys.maxsize))\"" - "%PYTHON%\\python.exe -c \"from msgpack import _packer, _unpacker\"" - "%PYTHON%\\Scripts\\py.test test" - - "build.cmd %PYTHON%\\python.exe setup.py bdist_wheel" + - "%PYTHON%\\python.exe setup.py bdist_wheel" after_test: # This step builds your wheels. - # Again, you only need build.cmd if you're building C extensions for - # 64-bit Python 3.3/3.4. And you need to use %PYTHON% to get the correct - # interpreter + # Again, you need to use %PYTHON% to get the correct interpreter artifacts: # bdist_wheel puts your built wheel in the dist directory diff --git a/build.cmd b/build.cmd deleted file mode 100644 index 243dc9a..0000000 --- a/build.cmd +++ /dev/null @@ -1,21 +0,0 @@ -@echo off -:: To build extensions for 64 bit Python 3, we need to configure environment -:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 4 -:: -:: More details at: -:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows - -IF "%DISTUTILS_USE_SDK%"=="1" ( - ECHO Configuring environment to build with MSVC on a 64bit architecture - ECHO Using Windows SDK 7.1 - "C:\Program Files\Microsoft SDKs\Windows\v7.1\Setup\WindowsSdkVer.exe" -q -version:v7.1 - CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 /release - SET MSSdk=1 - REM Need the following to allow tox to see the SDK compiler - SET TOX_TESTENV_PASSENV=DISTUTILS_USE_SDK MSSdk INCLUDE LIB -) ELSE ( - ECHO Using default MSVC build environment -) - -CALL %* diff --git a/docker/runtests.sh b/docker/runtests.sh index 0eea715..11ef9f4 100755 --- a/docker/runtests.sh +++ b/docker/runtests.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -x -for V in cp36-cp36m cp35-cp35m cp34-cp34m cp27-cp27m cp27-cp27mu; do +for V in cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do PYBIN=/opt/python/$V/bin $PYBIN/python setup.py install rm -rf build/ # Avoid lib build by narrow Python is used by wide python diff --git a/tox.ini b/tox.ini index b2ac336..68a2f53 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py33,py34,py35,py36}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86 +envlist = {py27,py35,py36}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86 [variants:pure] setenv= From 99341035f2f7f7b9d708c73f59e92277579abb0c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 21 Dec 2017 20:46:14 +0900 Subject: [PATCH 0906/1172] fix zero length raw can't be decoded. (#236) fix #234 --- msgpack/unpack_template.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index e1e08fe..525dea2 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -82,7 +82,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize const unsigned char* p = (unsigned char*)data + *off; const unsigned char* const pe = (unsigned char*)data + len; - const void* n = NULL; + const void* n = p; unsigned int trail = ctx->trail; unsigned int cs = ctx->cs; From 2eb6e75db1d4b4e30997aa88f9e904dc462a28da Mon Sep 17 00:00:00 2001 From: aaron jheng Date: Sun, 31 Dec 2017 10:52:50 +0800 Subject: [PATCH 0907/1172] add license info to metadata (#260) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 5c88397..5a26241 100755 --- a/setup.py +++ b/setup.py @@ -112,6 +112,7 @@ setup(name='msgpack-python', description=desc, long_description=long_desc, url='http://msgpack.org/', + license='Apache 2.0', classifiers=[ 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', From 0e2021d3a3d1218ca191f4e802df0af3bbfaa51f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 5 Jan 2018 19:16:14 +0900 Subject: [PATCH 0908/1172] Update changelog --- ChangeLog.rst | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 73ffc30..ed0f92e 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,24 +1,33 @@ 0.5.0 ====== -0.5 is important step toward 1.0. There are some deprecations. -Please read changes carefully. +There are some deprecations. Please read changes carefully. Changes ------- -* Drop Python 2.6 and 3.2 support +* Drop Python 2.6 and ~3.4 support. Python 2.7 and 3.5+ are supported. * Deprecate useless custom exceptions. Use ValueError instead of PackValueError, Exception instead of PackException and UnpackException, etc... See msgpack/exceptions.py -* Add `strict_types` option to packer. It can be used to serialize subclass of +* Add *strict_types* option to packer. It can be used to serialize subclass of builtin types. For example, when packing object which type is subclass of dict, - `default()` is called. + ``default()`` is called. ``default()`` is called for tuple too. * Pure Python implementation supports packing memoryview object. +* Support packing bytearray. + +* Add ``Unpacker.tell()``. And ``write_bytes`` option is deprecated. + + +Bugs fixed +---------- + +* Fixed zero length raw can't be decoded when encoding is specified. (#236) + 0.4.8 ===== From 43137d6bd2cc841af775a9c8132e72d284b119e3 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 5 Jan 2018 20:19:04 +0900 Subject: [PATCH 0909/1172] Deprecate write_bytes option in Unpacker. (#262) Fixes #197 --- msgpack/_unpacker.pyx | 3 +++ msgpack/fallback.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index dabc5f7..564749e 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -458,6 +458,9 @@ cdef class Unpacker(object): cdef object obj cdef Py_ssize_t prev_head + if write_bytes is not None: + PyErr_WarnEx(DeprecationWarning, "`write_bytes` option is deprecated. Use `.tell()` instead.", 1) + if self.buf_head >= self.buf_tail and self.file_like is not None: self.read_from_file() diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 28478ca..3c9c3b8 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -609,12 +609,14 @@ class Unpacker(object): def skip(self, write_bytes=None): self._unpack(EX_SKIP) if write_bytes is not None: + warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning) write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) self._consume() def unpack(self, write_bytes=None): ret = self._unpack(EX_CONSTRUCT) if write_bytes is not None: + warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning) write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) self._consume() return ret @@ -622,6 +624,7 @@ class Unpacker(object): def read_array_header(self, write_bytes=None): ret = self._unpack(EX_READ_ARRAY_HEADER) if write_bytes is not None: + warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning) write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) self._consume() return ret @@ -629,6 +632,7 @@ class Unpacker(object): def read_map_header(self, write_bytes=None): ret = self._unpack(EX_READ_MAP_HEADER) if write_bytes is not None: + warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning) write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) self._consume() return ret From 1979722ba2de84e68ae5992d33bc39461aa7b4b2 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 5 Jan 2018 20:58:14 +0900 Subject: [PATCH 0910/1172] Raise MemoryError when failed to grow buffer (#263) --- msgpack/pack.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/msgpack/pack.h b/msgpack/pack.h index d3aeff7..3bc21ea 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -48,7 +48,10 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ if (len + l > bs) { bs = (len + l) * 2; buf = (char*)PyMem_Realloc(buf, bs); - if (!buf) return -1; + if (!buf) { + PyErr_NoMemory(); + return -1; + } } memcpy(buf + len, data, l); len += l; From d0d3a403892106dfb809693f5e006a546cb55b83 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 6 Jan 2018 02:07:39 +0900 Subject: [PATCH 0911/1172] Warn about future use_bin_type change (#264) --- README.rst | 13 ++++++++----- msgpack/_packer.pyx | 12 ++++++++++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 3c16078..20353f5 100644 --- a/README.rst +++ b/README.rst @@ -60,7 +60,7 @@ msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with .. code-block:: pycon >>> import msgpack - >>> msgpack.packb([1, 2, 3]) + >>> msgpack.packb([1, 2, 3], use_bin_type=True) '\x93\x01\x02\x03' >>> msgpack.unpackb(_) [1, 2, 3] @@ -91,13 +91,13 @@ stream (or from bytes provided through its ``feed`` method). buf = BytesIO() for i in range(100): - buf.write(msgpack.packb(range(i))) + buf.write(msgpack.packb(range(i), use_bin_type=True)) buf.seek(0) unpacker = msgpack.Unpacker(buf) for unpacked in unpacker: - print unpacked + print(unpacked) Packing/unpacking of custom data type @@ -109,7 +109,6 @@ It is also possible to pack/unpack custom data types. Here is an example for .. code-block:: python import datetime - import msgpack useful_dict = { @@ -128,7 +127,7 @@ It is also possible to pack/unpack custom data types. Here is an example for return obj - packed_dict = msgpack.packb(useful_dict, default=encode_datetime) + packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True) this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime) ``Unpacker``'s ``object_hook`` callback receives a dict; the @@ -208,6 +207,10 @@ the packer. If you do so, it will use a non-standard type called **bin** to serialize byte arrays, and **raw** becomes to mean **str**. If you want to distinguish **bin** and **raw** in the unpacker, specify `encoding='utf-8'`. +**In future version, default value of ``use_bin_type`` will be changed to ``False``. +To avoid this change will break your code, you must specify it explicitly +even when you want to use old format.** + Note that Python 2 defaults to byte-arrays over Unicode strings: .. code-block:: pycon diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 5a81709..ebaeb65 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -2,6 +2,7 @@ #cython: embedsignature=True from cpython cimport * +from cpython.exc cimport PyErr_WarnEx from msgpack.exceptions import PackValueError, PackOverflowError from msgpack import ExtType @@ -76,6 +77,8 @@ cdef class Packer(object): :param bool use_bin_type: Use bin type introduced in msgpack spec 2.0 for bytes. It also enables str8 type for unicode. + Current default value is false, but it will be changed to true + in future version. You should specify it explicitly. :param bool strict_types: If set to true, types will be checked to be exact. Derived classes from serializeable types will not be serialized and will be @@ -103,12 +106,17 @@ cdef class Packer(object): self.pk.length = 0 def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', - use_single_float=False, bint autoreset=1, bint use_bin_type=0, + use_single_float=False, bint autoreset=1, use_bin_type=None, bint strict_types=0): + if use_bin_type is None: + PyErr_WarnEx( + FutureWarning, + "use_bin_type option is not specified. Default value of the option will be changed in future version.", + 1) self.use_float = use_single_float self.strict_types = strict_types self.autoreset = autoreset - self.pk.use_bin_type = use_bin_type + self.pk.use_bin_type = use_bin_type if default is not None: if not PyCallable_Check(default): raise TypeError("default must be a callable.") From 89e4f8b7b3a43fc84ea0dd278a4b8b54b1fac4bd Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 7 Jan 2018 01:57:47 +0900 Subject: [PATCH 0912/1172] Rename package name to msgpack --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 5a26241..419482b 100755 --- a/setup.py +++ b/setup.py @@ -102,7 +102,7 @@ with io.open('README.rst', encoding='utf-8') as f: long_desc = f.read() del f -setup(name='msgpack-python', +setup(name='msgpack', author='INADA Naoki', author_email='songofacandy@gmail.com', version=version_str, @@ -124,5 +124,5 @@ setup(name='msgpack-python', 'Programming Language :: Python :: Implementation :: PyPy', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', - ] - ) + ], +) From d720c42468c2e9cad2c04e28c23917187fa64412 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 7 Jan 2018 01:58:01 +0900 Subject: [PATCH 0913/1172] prepare 0.5 --- msgpack/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index 76bd8fb..f90cdc1 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 4, 8) +version = (0, 5, 0) From 9f4c12f29ce7a6d62423311ab12a031dc79c458a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 7 Jan 2018 01:59:14 +0900 Subject: [PATCH 0914/1172] Add transition package --- dummy/README | 2 ++ dummy/setup.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 dummy/README create mode 100644 dummy/setup.py diff --git a/dummy/README b/dummy/README new file mode 100644 index 0000000..bf1eeb6 --- /dev/null +++ b/dummy/README @@ -0,0 +1,2 @@ +This is dummy transition package for msgpak. +Install msgpack instead of msgpack-python. diff --git a/dummy/setup.py b/dummy/setup.py new file mode 100644 index 0000000..0f1e6e7 --- /dev/null +++ b/dummy/setup.py @@ -0,0 +1,31 @@ +from setuptools import setup, Extension + +long_desc = """\ +msgpack-python is renamed to just msgpack. + +Install msgpack by ``pip install msgpack``. +""" + + +setup(name='msgpack-python', + author='INADA Naoki', + author_email='songofacandy@gmail.com', + version="0.5.0", + description="Transition package for msgpack", + long_description=long_desc, + install_requires=["msgpack>=0.5"], + url='http://msgpack.org/', + license='Apache 2.0', + classifiers=[ + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + ], +) From 35fc29797036b9bf8619b726f7c7b231c508439e Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 7 Jan 2018 02:01:20 +0900 Subject: [PATCH 0915/1172] Update README --- README.rst | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/README.rst b/README.rst index 20353f5..f5fd233 100644 --- a/README.rst +++ b/README.rst @@ -23,12 +23,12 @@ Install :: - $ pip install msgpack-python + $ pip install msgpack PyPy ^^^^ -msgpack-python provides a pure Python implementation. PyPy can use this. +msgpack provides a pure Python implementation. PyPy can use this. Windows ^^^^^^^ @@ -249,17 +249,6 @@ To use the **ext** type, pass ``msgpack.ExtType`` object to packer. You can use it with ``default`` and ``ext_hook``. See below. -Note for msgpack-python 0.2.x users -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The msgpack-python release 0.3 has some incompatible changes. - -The default value of ``use_list`` keyword argument is ``True`` from 0.3. -You should pass the argument explicitly for backward compatibility. - -`Unpacker.unpack()` and some unpack methods now raises `OutOfData` -instead of `StopIteration`. -`StopIteration` is used for iterator protocol only. Note about performance ---------------------- From dbb827815aef9e26c54467436bc267c23b2eff81 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 7 Jan 2018 02:04:49 +0900 Subject: [PATCH 0916/1172] Update Cython version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 87f04da..cd54e6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython==0.25.2 +Cython==0.27.3 From 7c22d983f4aad34d612e6fb4dc84676fcb2bece9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 9 Jan 2018 13:17:47 +0900 Subject: [PATCH 0917/1172] Update README --- README.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.rst b/README.rst index f5fd233..fed1dfa 100644 --- a/README.rst +++ b/README.rst @@ -10,6 +10,20 @@ MessagePack for Python :target: https://msgpack-python.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status +Upgrading from msgpack-0.4 +-------------------------- + +TL;DR: When upgrading from msgpack-0.4 or earlier, don't do `pip install -U msgpack-python`. +Do `pip uninstall msgpack-python; pip install msgpack` instead. + +Package name on PyPI was changed to msgpack from 0.5. +I upload transitional package (msgpack-python 0.5 which depending on msgpack) +for smooth transition from msgpack-python to msgpack. + +Sadly, this doesn't work for upgrade install. After `pip install -U msgpack-python`, +msgpack is removed and `import msgpack` fail. + + What's this ----------- From 45c1a53d5aeab5799f76cfae02f064ba89098f7f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 9 Jan 2018 17:58:32 +0900 Subject: [PATCH 0918/1172] Update AppVeyor build (#267) --- appveyor.yml | 32 ++++++++++++++------------------ ci/runtests.bat | 7 +++++++ 2 files changed, 21 insertions(+), 18 deletions(-) create mode 100644 ci/runtests.bat diff --git a/appveyor.yml b/appveyor.yml index d581839..72b334a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,22 +1,12 @@ environment: - matrix: - # For Python versions available on Appveyor, see # http://www.appveyor.com/docs/installed-software#python - # The list here is complete (excluding Python 2.6, which - # isn't covered by this document) at the time of writing. - - - PYTHON: "C:\\Python27" - - PYTHON: "C:\\Python27-x64" - - PYTHON: "C:\\Python35" - - PYTHON: "C:\\Python35-x64" - PYTHON: "C:\\Python36" - - PYTHON: "C:\\Python36-x64" install: # We need wheel installed to build wheels - - "%PYTHON%\\python.exe -m pip install -U pip wheel pytest cython" + - "%PYTHON%\\python.exe -m pip install -U cython" - "%PYTHON%\\Scripts\\cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx" build: off @@ -26,12 +16,18 @@ test_script: # Note that you must use the environment variable %PYTHON% to refer to # the interpreter you're using - Appveyor does not do anything special # to put the Python version you want to use on PATH. - - "%PYTHON%\\python.exe setup.py build_ext -i" - - "%PYTHON%\\python.exe setup.py install" - - "%PYTHON%\\python.exe -c \"import sys; print(hex(sys.maxsize))\"" - - "%PYTHON%\\python.exe -c \"from msgpack import _packer, _unpacker\"" - - "%PYTHON%\\Scripts\\py.test test" - - "%PYTHON%\\python.exe setup.py bdist_wheel" + - set PYTHON="C:\\Python27" + - ci\\runtests.bat + - set PYTHON="C:\\Python27-x64" + - ci\\runtests.bat + - set PYTHON="C:\\Python35" + - ci\\runtests.bat + - set PYTHON="C:\\Python35-x64" + - ci\\runtests.bat + - set PYTHON="C:\\Python36" + - ci\\runtests.bat + - set PYTHON="C:\\Python36-x64" + - ci\\runtests.bat after_test: # This step builds your wheels. @@ -39,7 +35,7 @@ after_test: artifacts: # bdist_wheel puts your built wheel in the dist directory - - path: dist\* + - path: dist\*.whl #on_success: # You can use this step to upload your artifacts to a public website. diff --git a/ci/runtests.bat b/ci/runtests.bat new file mode 100644 index 0000000..9efea00 --- /dev/null +++ b/ci/runtests.bat @@ -0,0 +1,7 @@ +%PYTHON%\python.exe -m pip install -U pip wheel pytest +%PYTHON%\python.exe setup.py build_ext -i +%PYTHON%\python.exe setup.py install +%PYTHON%\python.exe -c "import sys; print(hex(sys.maxsize))" +%PYTHON%\python.exe -c "from msgpack import _packer, _unpacker" +%PYTHON%\python.exe -m pytest -v test +%PYTHON%\python.exe setup.py bdist_wheel From 676bbcd0eee2a6e8aece5239e380c12ea633375a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 9 Jan 2018 19:00:42 +0900 Subject: [PATCH 0919/1172] manylinux1: Add 3.6 and remove 3.4 --- docker/buildwheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh index b654e45..f586a8d 100644 --- a/docker/buildwheel.sh +++ b/docker/buildwheel.sh @@ -4,7 +4,7 @@ set -e -x ARCH=`uname -p` echo "arch=$ARCH" -for V in cp35-cp35m cp34-cp34m cp27-cp27m cp27-cp27mu; do +for V in cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do PYBIN=/opt/python/$V/bin rm -rf build/ # Avoid lib build by narrow Python is used by wide python $PYBIN/python setup.py bdist_wheel -p manylinux1_${ARCH} From e0934355c6534690d3c80ea8659d51c65a55ee0f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 9 Jan 2018 20:48:45 +0900 Subject: [PATCH 0920/1172] Update Makefile --- Makefile | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index f833bbc..84decd8 100644 --- a/Makefile +++ b/Makefile @@ -1,32 +1,27 @@ -.PHONY: test all python3 - +.PHONY: all all: cython python setup.py build_ext -i -f -doc-serve: all - cd docs && make serve - -doc: - cd docs && make zip - -upload-doc: - python setup.py upload_docs --upload-dir docs/_build/html - +.PHONY: cython cython: cython --cplus msgpack/*.pyx -python3: cython - python3 setup.py build_ext -i -f - +.PHONY: test test: - py.test test + py.test -v test + +.PHONY: serve-doc +serve-doc: all + cd docs && make serve .PHONY: clean clean: rm -rf build rm msgpack/*.so rm -rf msgpack/__pycache__ + rm -rf test/__pycache__ -build-manylinux1-wheel: +.PHONY: linux-wheel +linux-wheel: docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_i686 bash docker/buildwheel.sh docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_x86_64 bash docker/buildwheel.sh From ab66c272b03805fd16f0346238e8b7d1233b96c4 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 9 Jan 2018 22:03:06 +0900 Subject: [PATCH 0921/1172] Update README --- README.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index fed1dfa..d863328 100644 --- a/README.rst +++ b/README.rst @@ -148,6 +148,7 @@ It is also possible to pack/unpack custom data types. Here is an example for ``object_pairs_hook`` callback may instead be used to receive a list of key-value pairs. + Extended types ^^^^^^^^^^^^^^ @@ -170,7 +171,7 @@ It is also possible to pack/unpack custom data types using the **ext** type. ... return ExtType(code, data) ... >>> data = array.array('d', [1.2, 3.4]) - >>> packed = msgpack.packb(data, default=default) + >>> packed = msgpack.packb(data, default=default, use_bin_type=True) >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook) >>> data == unpacked True @@ -294,7 +295,7 @@ Test MessagePack uses `pytest` for testing. Run test with following command: - $ py.test + $ pytest -v test .. From 5be93786404d4e95de933d1bc64640402c3f2696 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 10 Jan 2018 02:48:08 +0900 Subject: [PATCH 0922/1172] Make msgpack-python deprecated clone of msgpack. --- README.rst | 4 ++-- dummy/README | 2 -- dummy/setup.py | 31 ------------------------------- setup.py | 11 ++++++++++- 4 files changed, 12 insertions(+), 36 deletions(-) delete mode 100644 dummy/README delete mode 100644 dummy/setup.py diff --git a/README.rst b/README.rst index d863328..42758b8 100644 --- a/README.rst +++ b/README.rst @@ -10,8 +10,8 @@ MessagePack for Python :target: https://msgpack-python.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status -Upgrading from msgpack-0.4 --------------------------- +IMPORTANT: Upgrading from msgpack-0.4 +-------------------------------------- TL;DR: When upgrading from msgpack-0.4 or earlier, don't do `pip install -U msgpack-python`. Do `pip uninstall msgpack-python; pip install msgpack` instead. diff --git a/dummy/README b/dummy/README deleted file mode 100644 index bf1eeb6..0000000 --- a/dummy/README +++ /dev/null @@ -1,2 +0,0 @@ -This is dummy transition package for msgpak. -Install msgpack instead of msgpack-python. diff --git a/dummy/setup.py b/dummy/setup.py deleted file mode 100644 index 0f1e6e7..0000000 --- a/dummy/setup.py +++ /dev/null @@ -1,31 +0,0 @@ -from setuptools import setup, Extension - -long_desc = """\ -msgpack-python is renamed to just msgpack. - -Install msgpack by ``pip install msgpack``. -""" - - -setup(name='msgpack-python', - author='INADA Naoki', - author_email='songofacandy@gmail.com', - version="0.5.0", - description="Transition package for msgpack", - long_description=long_desc, - install_requires=["msgpack>=0.5"], - url='http://msgpack.org/', - license='Apache 2.0', - classifiers=[ - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: Apache Software License', - ], -) diff --git a/setup.py b/setup.py index 419482b..6108587 100755 --- a/setup.py +++ b/setup.py @@ -9,6 +9,9 @@ from setuptools import setup, Extension from distutils.command.build_ext import build_ext +# for building transitional package. +TRANSITIONAL = False + class NoCython(Exception): pass @@ -102,7 +105,13 @@ with io.open('README.rst', encoding='utf-8') as f: long_desc = f.read() del f -setup(name='msgpack', +name = 'msgpack' + +if TRANSITIONAL: + name = 'msgpack-python' + long_desc = "This package is deprecated. Install msgpack instead." + +setup(name=name, author='INADA Naoki', author_email='songofacandy@gmail.com', version=version_str, From e0f2fd3af348e26b269d7eb90c74876d908aafca Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 10 Jan 2018 02:49:50 +0900 Subject: [PATCH 0923/1172] Fix README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 42758b8..01a8b2a 100644 --- a/README.rst +++ b/README.rst @@ -222,7 +222,7 @@ the packer. If you do so, it will use a non-standard type called **bin** to serialize byte arrays, and **raw** becomes to mean **str**. If you want to distinguish **bin** and **raw** in the unpacker, specify `encoding='utf-8'`. -**In future version, default value of ``use_bin_type`` will be changed to ``False``. +**In future version, default value of ``use_bin_type`` will be changed to ``True``. To avoid this change will break your code, you must specify it explicitly even when you want to use old format.** From 0112957bcff8e16dddd6cbc474bfe8a49f418fad Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 10 Jan 2018 02:54:59 +0900 Subject: [PATCH 0924/1172] Remove FutureWarning about use_bin_type option (#271) --- msgpack/_packer.pyx | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index ebaeb65..13a18f6 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -2,7 +2,7 @@ #cython: embedsignature=True from cpython cimport * -from cpython.exc cimport PyErr_WarnEx +#from cpython.exc cimport PyErr_WarnEx from msgpack.exceptions import PackValueError, PackOverflowError from msgpack import ExtType @@ -65,20 +65,20 @@ cdef class Packer(object): :param callable default: Convert user type to builtin type that Packer supports. See also simplejson's document. - :param str encoding: - Convert unicode to bytes with this encoding. (default: 'utf-8') - :param str unicode_errors: - Error handler for encoding unicode. (default: 'strict') + :param bool use_single_float: Use single precision float type for float. (default: False) + :param bool autoreset: Reset buffer after each pack and return its content as `bytes`. (default: True). If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. + :param bool use_bin_type: Use bin type introduced in msgpack spec 2.0 for bytes. It also enables str8 type for unicode. Current default value is false, but it will be changed to true in future version. You should specify it explicitly. + :param bool strict_types: If set to true, types will be checked to be exact. Derived classes from serializeable types will not be serialized and will be @@ -86,6 +86,11 @@ cdef class Packer(object): Additionally tuples will not be serialized as lists. This is useful when trying to implement accurate serialization for python types. + + :param str encoding: + (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8') + :param str unicode_errors: + (deprecated) Error handler for encoding unicode. (default: 'strict') """ cdef msgpack_packer pk cdef object _default @@ -106,17 +111,12 @@ cdef class Packer(object): self.pk.length = 0 def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', - use_single_float=False, bint autoreset=1, use_bin_type=None, - bint strict_types=0): - if use_bin_type is None: - PyErr_WarnEx( - FutureWarning, - "use_bin_type option is not specified. Default value of the option will be changed in future version.", - 1) + bint use_single_float=False, bint autoreset=True, bint use_bin_type=False, + bint strict_types=False): self.use_float = use_single_float self.strict_types = strict_types self.autoreset = autoreset - self.pk.use_bin_type = use_bin_type + self.pk.use_bin_type = use_bin_type if default is not None: if not PyCallable_Check(default): raise TypeError("default must be a callable.") From fc09da997ca323cb1a545478e9c1563d8db37ab1 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 10 Jan 2018 02:58:55 +0900 Subject: [PATCH 0925/1172] fallback: Update docstring. --- msgpack/_version.py | 2 +- msgpack/fallback.py | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index f90cdc1..ecba3d8 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 5, 0) +version = (0, 5, 1) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 3c9c3b8..5447b53 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -160,7 +160,7 @@ class Unpacker(object): If it is None (default), msgpack raw is deserialized to Python bytes. :param str unicode_errors: - Used for decoding msgpack raw with *encoding*. + (deprecated) Used for decoding msgpack raw with *encoding*. (default: `'strict'`) :param int max_buffer_size: @@ -656,18 +656,18 @@ class Packer(object): :param callable default: Convert user type to builtin type that Packer supports. See also simplejson's document. - :param str encoding: - Convert unicode to bytes with this encoding. (default: 'utf-8') - :param str unicode_errors: - Error handler for encoding unicode. (default: 'strict') + :param bool use_single_float: Use single precision float type for float. (default: False) + :param bool autoreset: Reset buffer after each pack and return its content as `bytes`. (default: True). If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. + :param bool use_bin_type: Use bin type introduced in msgpack spec 2.0 for bytes. It also enables str8 type for unicode. + :param bool strict_types: If set to true, types will be checked to be exact. Derived classes from serializeable types will not be serialized and will be @@ -675,6 +675,12 @@ class Packer(object): Additionally tuples will not be serialized as lists. This is useful when trying to implement accurate serialization for python types. + + :param str encoding: + (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8') + + :param str unicode_errors: + (deprecated) Error handler for encoding unicode. (default: 'strict') """ def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, autoreset=True, use_bin_type=False, From 50ea49c86f5aaff8bb1cd37778b50b13df83ba8f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 10 Jan 2018 03:04:54 +0900 Subject: [PATCH 0926/1172] Update doc --- docs/Makefile | 2 +- docs/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index 0869604..b09d884 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -153,7 +153,7 @@ doctest: "results in $(BUILDDIR)/doctest/output.txt." serve: html - cd _build/html && python3.3 -m http.server + cd _build/html && python3 -m http.server zip: html cd _build/html && zip -r ../../../msgpack-doc.zip . diff --git a/docs/conf.py b/docs/conf.py index 0f19fcc..47d745a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -49,7 +49,7 @@ copyright = u'2013, INADA Naoki' # # The short X.Y version. # The full version, including alpha/beta/rc tags. -version = release = '0.4' +version = release = '0.5' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 5534d0c7af0114db3d27f7b96c82a7fe22ce1e40 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 11 Jan 2018 17:02:41 +0900 Subject: [PATCH 0927/1172] Add raw_as_bytes option to Unpacker. (#265) --- Makefile | 3 +- README.rst | 78 ++++++++++++++++++++++++++++----------- ci/runtests.bat | 4 +- msgpack/_packer.pyx | 18 ++++++--- msgpack/_unpacker.pyx | 81 ++++++++++++++++++++++++++--------------- msgpack/fallback.py | 52 +++++++++++++++++++++++--- msgpack/unpack.h | 12 ++++-- test/test_limits.py | 4 +- test/test_pack.py | 22 +++++------ test/test_stricttype.py | 8 ++-- test/test_unpack.py | 10 ++--- 11 files changed, 199 insertions(+), 93 deletions(-) diff --git a/Makefile b/Makefile index 84decd8..6a9906c 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,8 @@ cython: .PHONY: test test: - py.test -v test + pytest -v test + MSGPACK_PUREPYTHON=1 pytest -v test .PHONY: serve-doc serve-doc: all diff --git a/README.rst b/README.rst index 01a8b2a..a5038db 100644 --- a/README.rst +++ b/README.rst @@ -10,8 +10,21 @@ MessagePack for Python :target: https://msgpack-python.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status -IMPORTANT: Upgrading from msgpack-0.4 --------------------------------------- + +What's this +----------- + +`MessagePack `_ is an efficient binary serialization format. +It lets you exchange data among multiple languages like JSON. +But it's faster and smaller. +This package provides CPython bindings for reading and writing MessagePack data. + + +Very important notes for existing users +--------------------------------------- + +PyPI package name +^^^^^^^^^^^^^^^^^ TL;DR: When upgrading from msgpack-0.4 or earlier, don't do `pip install -U msgpack-python`. Do `pip uninstall msgpack-python; pip install msgpack` instead. @@ -24,13 +37,37 @@ Sadly, this doesn't work for upgrade install. After `pip install -U msgpack-pyt msgpack is removed and `import msgpack` fail. -What's this ------------ +Deprecating encoding option +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +encoding and unicode_errors options are deprecated. + +In case of packer, use UTF-8 always. Storing other than UTF-8 is not recommended. + +For backward compatibility, you can use ``use_bin_type=False`` and pack ``bytes`` +object into msgpack raw type. + +In case of unpacker, there is new ``raw_as_bytes`` option. It is ``True`` by default +for backward compatibility, but it is changed to ``False`` in near future. +You can use ``raw_as_bytes=False`` instead of ``encoding='utf-8'``. + +Planned backward incompatible changes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When msgpack 1.0, I planning these breaking changes: + +* packer and unpacker: Remove ``encoding`` and ``unicode_errors`` option. +* packer: Change default of ``use_bin_type`` option from False to True. +* unpacker: Change default of ``raw_as_bytes`` option from True to False. +* unpacker: Reduce all ``max_xxx_len`` options for typical usage. +* unpacker: Remove ``write_bytes`` option from all methods. + +To avoid these breaking changes breaks your application, please: + +* Don't use deprecated options. +* Pass ``use_bin_type`` and ``raw_as_bytes`` options explicitly. +* If your application handle large (>1MB) data, specify ``max_xxx_len`` options too. -`MessagePack `_ is an efficient binary serialization format. -It lets you exchange data among multiple languages like JSON. -But it's faster and smaller. -This package provides CPython bindings for reading and writing MessagePack data. Install ------- @@ -76,14 +113,14 @@ msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with >>> import msgpack >>> msgpack.packb([1, 2, 3], use_bin_type=True) '\x93\x01\x02\x03' - >>> msgpack.unpackb(_) + >>> msgpack.unpackb(_, raw_as_bytes=False) [1, 2, 3] ``unpack`` unpacks msgpack's array to Python's list, but can also unpack to tuple: .. code-block:: pycon - >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False) + >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw_as_bytes=False) (1, 2, 3) You should always specify the ``use_list`` keyword argument for backward compatibility. @@ -109,7 +146,7 @@ stream (or from bytes provided through its ``feed`` method). buf.seek(0) - unpacker = msgpack.Unpacker(buf) + unpacker = msgpack.Unpacker(buf, raw_as_bytes=False) for unpacked in unpacker: print(unpacked) @@ -142,7 +179,7 @@ It is also possible to pack/unpack custom data types. Here is an example for packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True) - this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime) + this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw_as_bytes=False) ``Unpacker``'s ``object_hook`` callback receives a dict; the ``object_pairs_hook`` callback may instead be used to receive a list of @@ -172,7 +209,7 @@ It is also possible to pack/unpack custom data types using the **ext** type. ... >>> data = array.array('d', [1.2, 3.4]) >>> packed = msgpack.packb(data, default=default, use_bin_type=True) - >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook) + >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw_as_bytes=False) >>> data == unpacked True @@ -217,14 +254,10 @@ Early versions of msgpack didn't distinguish string and binary types (like Pytho The type for representing both string and binary types was named **raw**. For backward compatibility reasons, msgpack-python will still default all -strings to byte strings, unless you specify the `use_bin_type=True` option in +strings to byte strings, unless you specify the ``use_bin_type=True`` option in the packer. If you do so, it will use a non-standard type called **bin** to serialize byte arrays, and **raw** becomes to mean **str**. If you want to -distinguish **bin** and **raw** in the unpacker, specify `encoding='utf-8'`. - -**In future version, default value of ``use_bin_type`` will be changed to ``True``. -To avoid this change will break your code, you must specify it explicitly -even when you want to use old format.** +distinguish **bin** and **raw** in the unpacker, specify ``raw_as_bytes=False``. Note that Python 2 defaults to byte-arrays over Unicode strings: @@ -234,7 +267,7 @@ Note that Python 2 defaults to byte-arrays over Unicode strings: >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'])) ['spam', 'eggs'] >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), - encoding='utf-8') + raw_as_bytes=False) ['spam', u'eggs'] This is the same code in Python 3 (same behaviour, but Python 3 has a @@ -246,7 +279,7 @@ different default): >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'])) [b'spam', b'eggs'] >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), - encoding='utf-8') + raw_as_bytes=False) [b'spam', 'eggs'] @@ -277,6 +310,7 @@ You can use ``gc.disable()`` when unpacking large message. use_list option ^^^^^^^^^^^^^^^ + List is the default sequence type of Python. But tuple is lighter than list. You can use ``use_list=False`` while unpacking when performance is important. @@ -295,7 +329,7 @@ Test MessagePack uses `pytest` for testing. Run test with following command: - $ pytest -v test + $ make test .. diff --git a/ci/runtests.bat b/ci/runtests.bat index 9efea00..0240467 100644 --- a/ci/runtests.bat +++ b/ci/runtests.bat @@ -3,5 +3,7 @@ %PYTHON%\python.exe setup.py install %PYTHON%\python.exe -c "import sys; print(hex(sys.maxsize))" %PYTHON%\python.exe -c "from msgpack import _packer, _unpacker" -%PYTHON%\python.exe -m pytest -v test %PYTHON%\python.exe setup.py bdist_wheel +%PYTHON%\python.exe -m pytest -v test +SET EL=%ERRORLEVEL% +exit /b %EL% diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 13a18f6..39da91b 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -2,7 +2,7 @@ #cython: embedsignature=True from cpython cimport * -#from cpython.exc cimport PyErr_WarnEx +from cpython.exc cimport PyErr_WarnEx from msgpack.exceptions import PackValueError, PackOverflowError from msgpack import ExtType @@ -39,7 +39,7 @@ cdef extern from "pack.h": int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l) cdef int DEFAULT_RECURSE_LIMIT=511 -cdef size_t ITEM_LIMIT = (2**32)-1 +cdef long long ITEM_LIMIT = (2**32)-1 cdef inline int PyBytesLike_Check(object o): @@ -110,9 +110,13 @@ cdef class Packer(object): self.pk.buf_size = buf_size self.pk.length = 0 - def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', + def __init__(self, default=None, encoding=None, unicode_errors=None, bint use_single_float=False, bint autoreset=True, bint use_bin_type=False, bint strict_types=False): + if encoding is not None: + PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated.", 1) + if unicode_errors is not None: + PyErr_WarnEx(PendingDeprecationWarning, "unicode_errors is deprecated.", 1) self.use_float = use_single_float self.strict_types = strict_types self.autoreset = autoreset @@ -122,7 +126,7 @@ cdef class Packer(object): raise TypeError("default must be a callable.") self._default = default if encoding is None: - self.encoding = NULL + self.encoding = 'utf_8' self.unicode_errors = NULL else: if isinstance(encoding, unicode): @@ -134,7 +138,8 @@ cdef class Packer(object): self._berrors = unicode_errors.encode('ascii') else: self._berrors = unicode_errors - self.unicode_errors = PyBytes_AsString(self._berrors) + if self._berrors is not None: + self.unicode_errors = PyBytes_AsString(self._berrors) def __dealloc__(self): PyMem_Free(self.pk.buf) @@ -149,7 +154,7 @@ cdef class Packer(object): cdef char* rawval cdef int ret cdef dict d - cdef size_t L + cdef Py_ssize_t L cdef int default_used = 0 cdef bint strict_types = self.strict_types cdef Py_buffer view @@ -203,6 +208,7 @@ cdef class Packer(object): elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): if not self.encoding: raise TypeError("Can't encode unicode string: no encoding is specified") + #TODO: Use faster API for UTF-8 o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) L = len(o) if L > ITEM_LIMIT: diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 564749e..b796d04 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -43,8 +43,9 @@ from msgpack import ExtType cdef extern from "unpack.h": ctypedef struct msgpack_user: bint use_list - PyObject* object_hook + bint raw_as_bytes bint has_pairs_hook # call object_hook with k-v pairs + PyObject* object_hook PyObject* list_hook PyObject* ext_hook char *encoding @@ -73,12 +74,14 @@ cdef extern from "unpack.h": cdef inline init_ctx(unpack_context *ctx, object object_hook, object object_pairs_hook, object list_hook, object ext_hook, - bint use_list, char* encoding, char* unicode_errors, + bint use_list, bint raw_as_bytes, + char* encoding, char* unicode_errors, Py_ssize_t max_str_len, Py_ssize_t max_bin_len, Py_ssize_t max_array_len, Py_ssize_t max_map_len, Py_ssize_t max_ext_len): unpack_init(ctx) ctx.user.use_list = use_list + ctx.user.raw_as_bytes = raw_as_bytes ctx.user.object_hook = ctx.user.list_hook = NULL ctx.user.max_str_len = max_str_len ctx.user.max_bin_len = max_bin_len @@ -155,7 +158,8 @@ cdef inline int get_data_from_buffer(object obj, return 1 def unpackb(object packed, object object_hook=None, object list_hook=None, - bint use_list=1, encoding=None, unicode_errors="strict", + bint use_list=True, bint raw_as_bytes=True, + encoding=None, unicode_errors="strict", object_pairs_hook=None, ext_hook=ExtType, Py_ssize_t max_str_len=2147483647, # 2**32-1 Py_ssize_t max_bin_len=2147483647, @@ -180,21 +184,26 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef char* cerr = NULL cdef int new_protocol = 0 + if encoding is not None: + PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw_as_bytes=False instead.", 1) + if isinstance(encoding, unicode): + encoding = encoding.encode('ascii') + elif not isinstance(encoding, bytes): + raise TypeError("encoding should be bytes or unicode") + cenc = PyBytes_AsString(encoding) + + if unicode_errors is not None: + PyErr_WarnEx(PendingDeprecationWarning, "unicode_errors is deprecated", 1) + if isinstance(unicode_errors, unicode): + unicode_errors = unicode_errors.encode('ascii') + elif not isinstance(unicode_errors, bytes): + raise TypeError("unicode_errors should be bytes or unicode") + cerr = PyBytes_AsString(unicode_errors) + get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol) - try: - if encoding is not None: - if isinstance(encoding, unicode): - encoding = encoding.encode('ascii') - cenc = PyBytes_AsString(encoding) - - if unicode_errors is not None: - if isinstance(unicode_errors, unicode): - unicode_errors = unicode_errors.encode('ascii') - cerr = PyBytes_AsString(unicode_errors) - init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook, - use_list, cenc, cerr, + use_list, raw_as_bytes, cenc, cerr, max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) ret = unpack_construct(&ctx, buf, buf_len, &off) finally: @@ -252,6 +261,16 @@ cdef class Unpacker(object): If true, unpack msgpack array to Python list. Otherwise, unpack to Python tuple. (default: True) + :param bool raw_as_bytes: + If true, unpack msgpack raw to Python bytes (default). + Otherwise, unpack to Python str (or unicode on Python 2) by decoding + with UTF-8 encoding (recommended). + Currently, the default is true, but it will be changed to false in + near future. So you must specify it explicitly for keeping backward + compatibility. + + *encoding* option which is deprecated overrides this option. + :param callable object_hook: When specified, it should be callable. Unpacker calls it with a dict argument after unpacking msgpack map. @@ -262,14 +281,6 @@ cdef class Unpacker(object): Unpacker calls it with a list of key-value pairs after unpacking msgpack map. (See also simplejson) - :param str encoding: - Encoding used for decoding msgpack raw. - If it is None (default), msgpack raw is deserialized to Python bytes. - - :param str unicode_errors: - Used for decoding msgpack raw with *encoding*. - (default: `'strict'`) - :param int max_buffer_size: Limits size of data waiting unpacked. 0 means system's INT_MAX (default). Raises `BufferFull` exception when it is insufficient. @@ -287,16 +298,25 @@ cdef class Unpacker(object): :param int max_map_len: Limits max length of map. (default: 2**31-1) + :param str encoding: + Deprecated, use raw_as_bytes instead. + Encoding used for decoding msgpack raw. + If it is None (default), msgpack raw is deserialized to Python bytes. - example of streaming deserialize from file-like object:: + :param str unicode_errors: + Deprecated. Used for decoding msgpack raw with *encoding*. + (default: `'strict'`) - unpacker = Unpacker(file_like) + + Example of streaming deserialize from file-like object:: + + unpacker = Unpacker(file_like, raw_as_bytes=False) for o in unpacker: process(o) - example of streaming deserialize from socket:: + Example of streaming deserialize from socket:: - unpacker = Unpacker() + unpacker = Unpacker(raw_as_bytes=False) while True: buf = sock.recv(1024**2) if not buf: @@ -324,7 +344,8 @@ cdef class Unpacker(object): PyMem_Free(self.buf) self.buf = NULL - def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1, + def __init__(self, file_like=None, Py_ssize_t read_size=0, + bint use_list=True, bint raw_as_bytes=True, object object_hook=None, object object_pairs_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, object ext_hook=ExtType, @@ -363,6 +384,7 @@ cdef class Unpacker(object): self.stream_offset = 0 if encoding is not None: + PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw_as_bytes=False instead.", 1) if isinstance(encoding, unicode): self.encoding = encoding.encode('ascii') elif isinstance(encoding, bytes): @@ -372,6 +394,7 @@ cdef class Unpacker(object): cenc = PyBytes_AsString(self.encoding) if unicode_errors is not None: + PyErr_WarnEx(PendingDeprecationWarning, "unicode_errors is deprecated", 1) if isinstance(unicode_errors, unicode): self.unicode_errors = unicode_errors.encode('ascii') elif isinstance(unicode_errors, bytes): @@ -381,7 +404,7 @@ cdef class Unpacker(object): cerr = PyBytes_AsString(self.unicode_errors) init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, - ext_hook, use_list, cenc, cerr, + ext_hook, use_list, raw_as_bytes, cenc, cerr, max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 5447b53..d95f621 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -145,6 +145,16 @@ class Unpacker(object): If true, unpack msgpack array to Python list. Otherwise, unpack to Python tuple. (default: True) + :param bool raw_as_bytes: + If true, unpack msgpack raw to Python bytes (default). + Otherwise, unpack to Python str (or unicode on Python 2) by decoding + with UTF-8 encoding (recommended). + Currently, the default is true, but it will be changed to false in + near future. So you must specify it explicitly for keeping backward + compatibility. + + *encoding* option which is deprecated overrides this option. + :param callable object_hook: When specified, it should be callable. Unpacker calls it with a dict argument after unpacking msgpack map. @@ -183,13 +193,13 @@ class Unpacker(object): example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like) + unpacker = Unpacker(file_like, raw_as_bytes=False) for o in unpacker: process(o) example of streaming deserialize from socket:: - unpacker = Unpacker() + unpacker = Unpacker(raw_as_bytes=False) while True: buf = sock.recv(1024**2) if not buf: @@ -199,15 +209,28 @@ class Unpacker(object): process(o) """ - def __init__(self, file_like=None, read_size=0, use_list=True, + def __init__(self, file_like=None, read_size=0, use_list=True, raw_as_bytes=True, object_hook=None, object_pairs_hook=None, list_hook=None, - encoding=None, unicode_errors='strict', max_buffer_size=0, + encoding=None, unicode_errors=None, max_buffer_size=0, ext_hook=ExtType, max_str_len=2147483647, # 2**32-1 max_bin_len=2147483647, max_array_len=2147483647, max_map_len=2147483647, max_ext_len=2147483647): + + if encoding is not None: + warnings.warn( + "encoding is deprecated, Use raw_as_bytes=False instead.", + PendingDeprecationWarning) + + if unicode_errors is not None: + warnings.warn( + "unicode_errors is deprecated.", + PendingDeprecationWarning) + else: + unicode_errors = 'strict' + if file_like is None: self._feeding = True else: @@ -234,6 +257,7 @@ class Unpacker(object): if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") self._read_size = read_size or min(self._max_buffer_size, 16*1024) + self._raw_as_bytes = bool(raw_as_bytes) self._encoding = encoding self._unicode_errors = unicode_errors self._use_list = use_list @@ -582,8 +606,10 @@ class Unpacker(object): if typ == TYPE_RAW: if self._encoding is not None: obj = obj.decode(self._encoding, self._unicode_errors) - else: + elif self._raw_as_bytes: obj = bytes(obj) + else: + obj = obj.decode('utf_8') return obj if typ == TYPE_EXT: return self._ext_hook(n, bytes(obj)) @@ -682,9 +708,23 @@ class Packer(object): :param str unicode_errors: (deprecated) Error handler for encoding unicode. (default: 'strict') """ - def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', + def __init__(self, default=None, encoding=None, unicode_errors=None, use_single_float=False, autoreset=True, use_bin_type=False, strict_types=False): + if encoding is None: + encoding = 'utf_8' + else: + warnings.warn( + "encoding is deprecated, Use raw_as_bytes=False instead.", + PendingDeprecationWarning) + + if unicode_errors is None: + unicode_errors = 'strict' + else: + warnings.warn( + "unicode_errors is deprecated.", + PendingDeprecationWarning) + self._strict_types = strict_types self._use_float = use_single_float self._autoreset = autoreset diff --git a/msgpack/unpack.h b/msgpack/unpack.h index da2cfb6..8c2fc46 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -20,9 +20,10 @@ #include "unpack_define.h" typedef struct unpack_user { - int use_list; - PyObject *object_hook; + bool use_list; + bool raw_as_bytes; bool has_pairs_hook; + PyObject *object_hook; PyObject *list_hook; PyObject *ext_hook; const char *encoding; @@ -225,10 +226,13 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* } PyObject *py; - if(u->encoding) { + + if (u->encoding) { py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors); - } else { + } else if (u->raw_as_bytes) { py = PyBytes_FromStringAndSize(p, l); + } else { + py = PyUnicode_DecodeUTF8(p, l, NULL); } if (!py) return -1; diff --git a/test/test_limits.py b/test/test_limits.py index 197ef46..3febc30 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -39,11 +39,11 @@ def test_max_str_len(): d = 'x' * 3 packed = packb(d) - unpacker = Unpacker(max_str_len=3, encoding='utf-8') + unpacker = Unpacker(max_str_len=3, raw_as_bytes=False) unpacker.feed(packed) assert unpacker.unpack() == d - unpacker = Unpacker(max_str_len=2, encoding='utf-8') + unpacker = Unpacker(max_str_len=2, raw_as_bytes=False) with pytest.raises(UnpackValueError): unpacker.feed(packed) unpacker.unpack() diff --git a/test/test_pack.py b/test/test_pack.py index ac93103..29f5887 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -31,14 +31,14 @@ def testPack(): def testPackUnicode(): test_data = ["", "abcd", ["defgh"], "РуÑÑкий текÑÑ‚"] for td in test_data: - re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8') + re = unpackb(packb(td), use_list=1, raw_as_bytes=False) assert re == td - packer = Packer(encoding='utf-8') + packer = Packer() data = packer.pack(td) - re = Unpacker(BytesIO(data), encoding=str('utf-8'), use_list=1).unpack() + re = Unpacker(BytesIO(data), raw_as_bytes=False, use_list=1).unpack() assert re == td -def testPackUTF32(): +def testPackUTF32(): # deprecated try: test_data = [ "", @@ -66,26 +66,22 @@ def testPackByteArrays(): for td in test_data: check(td) -def testIgnoreUnicodeErrors(): +def testIgnoreUnicodeErrors(): # deprecated re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1) assert re == "abcdef" def testStrictUnicodeUnpack(): with raises(UnicodeDecodeError): - unpackb(packb(b'abc\xeddef'), encoding='utf-8', use_list=1) + unpackb(packb(b'abc\xeddef'), raw_as_bytes=False, use_list=1) -def testStrictUnicodePack(): +def testStrictUnicodePack(): # deprecated with raises(UnicodeEncodeError): packb("abc\xeddef", encoding='ascii', unicode_errors='strict') -def testIgnoreErrorsPack(): - re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1) +def testIgnoreErrorsPack(): # deprecated + re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw_as_bytes=False, use_list=1) assert re == "abcdef" -def testNoEncoding(): - with raises(TypeError): - packb("abc", encoding=None) - def testDecodeBinary(): re = unpackb(packb(b"abc"), encoding=None, use_list=1) assert re == b"abc" diff --git a/test/test_stricttype.py b/test/test_stricttype.py index 0f865c8..13239f1 100644 --- a/test/test_stricttype.py +++ b/test/test_stricttype.py @@ -11,7 +11,7 @@ def test_namedtuple(): return dict(o._asdict()) raise TypeError('Unsupported type %s' % (type(o),)) packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default) - unpacked = unpackb(packed, encoding='utf-8') + unpacked = unpackb(packed, raw_as_bytes=False) assert unpacked == {'foo': 1, 'bar': 42} @@ -32,7 +32,7 @@ def test_tuple(): return o data = packb(t, strict_types=True, use_bin_type=True, default=default) - expected = unpackb(data, encoding='utf-8', object_hook=convert) + expected = unpackb(data, raw_as_bytes=False, object_hook=convert) assert expected == t @@ -53,10 +53,10 @@ def test_tuple_ext(): def convert(code, payload): if code == MSGPACK_EXT_TYPE_TUPLE: # Unpack and convert to tuple - return tuple(unpackb(payload, encoding='utf-8', ext_hook=convert)) + return tuple(unpackb(payload, raw_as_bytes=False, ext_hook=convert)) raise ValueError('Unknown Ext code {}'.format(code)) data = packb(t, strict_types=True, use_bin_type=True, default=default) - expected = unpackb(data, encoding='utf-8', ext_hook=convert) + expected = unpackb(data, raw_as_bytes=False, ext_hook=convert) assert expected == t diff --git a/test/test_unpack.py b/test/test_unpack.py index c0d711c..143f999 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -47,8 +47,8 @@ def test_unpacker_ext_hook(): class MyUnpacker(Unpacker): def __init__(self): - super(MyUnpacker, self).__init__(ext_hook=self._hook, - encoding='utf-8') + super(MyUnpacker, self).__init__( + ext_hook=self._hook, raw_as_bytes=False) def _hook(self, code, data): if code == 1: @@ -57,11 +57,11 @@ def test_unpacker_ext_hook(): return ExtType(code, data) unpacker = MyUnpacker() - unpacker.feed(packb({'a': 1}, encoding='utf-8')) + unpacker.feed(packb({'a': 1})) assert unpacker.unpack() == {'a': 1} - unpacker.feed(packb({'a': ExtType(1, b'123')}, encoding='utf-8')) + unpacker.feed(packb({'a': ExtType(1, b'123')})) assert unpacker.unpack() == {'a': 123} - unpacker.feed(packb({'a': ExtType(2, b'321')}, encoding='utf-8')) + unpacker.feed(packb({'a': ExtType(2, b'321')})) assert unpacker.unpack() == {'a': ExtType(2, b'321')} From 60ef3879d792ec92480cf9d6d610951657c2e8c7 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 11 Jan 2018 19:41:05 +0900 Subject: [PATCH 0928/1172] packer: Use PyUnicode_AsUTF8AndSize() for utf-8 (#272) --- docker/runtests.sh | 2 +- msgpack/_packer.pyx | 36 +++++++++++++++++++++------------- msgpack/pack.h | 47 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 14 deletions(-) diff --git a/docker/runtests.sh b/docker/runtests.sh index 11ef9f4..113b630 100755 --- a/docker/runtests.sh +++ b/docker/runtests.sh @@ -9,6 +9,6 @@ for V in cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do pushd test # prevent importing msgpack package in current directory. $PYBIN/python -c 'import sys; print(hex(sys.maxsize))' $PYBIN/python -c 'from msgpack import _packer, _unpacker' - $PYBIN/py.test -v + $PYBIN/pytest -v . popd done diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 39da91b..a4913ab 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -13,6 +13,7 @@ cdef extern from "Python.h": int PyMemoryView_Check(object obj) int PyByteArray_Check(object obj) int PyByteArray_CheckExact(object obj) + char* PyUnicode_AsUTF8AndSize(object obj, Py_ssize_t *l) except NULL cdef extern from "pack.h": @@ -37,6 +38,7 @@ cdef extern from "pack.h": int msgpack_pack_bin(msgpack_packer* pk, size_t l) int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l) + int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit) cdef int DEFAULT_RECURSE_LIMIT=511 cdef long long ITEM_LIMIT = (2**32)-1 @@ -126,8 +128,12 @@ cdef class Packer(object): raise TypeError("default must be a callable.") self._default = default if encoding is None: - self.encoding = 'utf_8' - self.unicode_errors = NULL + if unicode_errors is None: + self.encoding = NULL + self.unicode_errors = NULL + else: + self.encoding = "utf_8" + self.unicode_errors = unicode_errors else: if isinstance(encoding, unicode): self._bencoding = encoding.encode('ascii') @@ -140,6 +146,8 @@ cdef class Packer(object): self._berrors = unicode_errors if self._berrors is not None: self.unicode_errors = PyBytes_AsString(self._berrors) + else: + self.unicode_errors = NULL def __dealloc__(self): PyMem_Free(self.pk.buf) @@ -206,17 +214,19 @@ cdef class Packer(object): if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, L) elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): - if not self.encoding: - raise TypeError("Can't encode unicode string: no encoding is specified") - #TODO: Use faster API for UTF-8 - o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) - L = len(o) - if L > ITEM_LIMIT: - raise PackValueError("unicode string is too large") - rawval = o - ret = msgpack_pack_raw(&self.pk, L) - if ret == 0: - ret = msgpack_pack_raw_body(&self.pk, rawval, L) + if self.encoding == NULL: + ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); + if ret == -2: + raise PackValueError("unicode string is too large") + else: + o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) + L = len(o) + if L > ITEM_LIMIT: + raise PackValueError("unicode string is too large") + ret = msgpack_pack_raw(&self.pk, L) + if ret == 0: + rawval = o + ret = msgpack_pack_raw_body(&self.pk, rawval, L) elif PyDict_CheckExact(o): d = o L = len(d) diff --git a/msgpack/pack.h b/msgpack/pack.h index 3bc21ea..4f3ce1d 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -67,6 +67,53 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ #include "pack_template.h" +// return -2 when o is too long +static inline int +msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit) +{ +#if PY_MAJOR_VERSION >= 3 + assert(PyUnicode_Check(o)); + + Py_ssize_t len; + const char* buf = PyUnicode_AsUTF8AndSize(o, &len); + if (buf == NULL) + return -1; + + if (len > limit) { + return -2; + } + + int ret = msgpack_pack_raw(pk, len); + if (ret) return ret; + + return msgpack_pack_raw_body(pk, buf, len); +#else + PyObject *bytes; + Py_ssize_t len; + int ret; + + // py2 + bytes = PyUnicode_AsUTF8String(o); + if (bytes == NULL) + return -1; + + len = PyString_GET_SIZE(bytes); + if (len > limit) { + Py_DECREF(bytes); + return -2; + } + + ret = msgpack_pack_raw(pk, len); + if (ret) { + Py_DECREF(bytes); + return -1; + } + ret = msgpack_pack_raw_body(pk, PyString_AS_STRING(bytes), len); + Py_DECREF(bytes); + return ret; +#endif +} + #ifdef __cplusplus } #endif From d9ec8fc905fc9ed37c86700f794adeb883b4f5ea Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 11 Jan 2018 23:50:41 +0900 Subject: [PATCH 0929/1172] Packer.pack() reset buffer on exception (#274) fixes #210 --- msgpack/_packer.pyx | 12 +++++++----- msgpack/fallback.py | 6 +++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index a4913ab..35e5a9d 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -289,11 +289,13 @@ cdef class Packer(object): cpdef pack(self, object obj): cdef int ret - ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) - if ret == -1: - raise MemoryError - elif ret: # should not happen. - raise TypeError + try: + ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) + except: + self.pk.length = 0 + raise + if ret: # should not happen. + raise RuntimeError("internal error") if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 diff --git a/msgpack/fallback.py b/msgpack/fallback.py index d95f621..675ee8a 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -848,7 +848,11 @@ class Packer(object): raise TypeError("Cannot serialize %r" % (obj, )) def pack(self, obj): - self._pack(obj) + try: + self._pack(obj) + except: + self._buffer = StringIO() # force reset + raise ret = self._buffer.getvalue() if self._autoreset: self._buffer = StringIO() From 5569a4efcdc913d343eaff4e55c9b19fafde4268 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 12 Jan 2018 19:22:36 +0900 Subject: [PATCH 0930/1172] s/raw_as_bytes/raw/g (#276) fixes #273 --- Makefile | 3 ++- README.rst | 24 ++++++++++++------------ msgpack/_unpacker.pyx | 26 +++++++++++++------------- msgpack/fallback.py | 16 ++++++++-------- msgpack/unpack.h | 4 ++-- test/test_limits.py | 4 ++-- test/test_pack.py | 8 ++++---- test/test_stricttype.py | 8 ++++---- test/test_unpack.py | 2 +- 9 files changed, 48 insertions(+), 47 deletions(-) diff --git a/Makefile b/Makefile index 6a9906c..4030080 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,8 @@ serve-doc: all .PHONY: clean clean: rm -rf build - rm msgpack/*.so + rm -f msgpack/_packer.cpp + rm -f msgpack/_unpacker.cpp rm -rf msgpack/__pycache__ rm -rf test/__pycache__ diff --git a/README.rst b/README.rst index a5038db..8925a65 100644 --- a/README.rst +++ b/README.rst @@ -47,9 +47,9 @@ In case of packer, use UTF-8 always. Storing other than UTF-8 is not recommende For backward compatibility, you can use ``use_bin_type=False`` and pack ``bytes`` object into msgpack raw type. -In case of unpacker, there is new ``raw_as_bytes`` option. It is ``True`` by default +In case of unpacker, there is new ``raw`` option. It is ``True`` by default for backward compatibility, but it is changed to ``False`` in near future. -You can use ``raw_as_bytes=False`` instead of ``encoding='utf-8'``. +You can use ``raw=False`` instead of ``encoding='utf-8'``. Planned backward incompatible changes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -58,14 +58,14 @@ When msgpack 1.0, I planning these breaking changes: * packer and unpacker: Remove ``encoding`` and ``unicode_errors`` option. * packer: Change default of ``use_bin_type`` option from False to True. -* unpacker: Change default of ``raw_as_bytes`` option from True to False. +* unpacker: Change default of ``raw`` option from True to False. * unpacker: Reduce all ``max_xxx_len`` options for typical usage. * unpacker: Remove ``write_bytes`` option from all methods. To avoid these breaking changes breaks your application, please: * Don't use deprecated options. -* Pass ``use_bin_type`` and ``raw_as_bytes`` options explicitly. +* Pass ``use_bin_type`` and ``raw`` options explicitly. * If your application handle large (>1MB) data, specify ``max_xxx_len`` options too. @@ -113,14 +113,14 @@ msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with >>> import msgpack >>> msgpack.packb([1, 2, 3], use_bin_type=True) '\x93\x01\x02\x03' - >>> msgpack.unpackb(_, raw_as_bytes=False) + >>> msgpack.unpackb(_, raw=False) [1, 2, 3] ``unpack`` unpacks msgpack's array to Python's list, but can also unpack to tuple: .. code-block:: pycon - >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw_as_bytes=False) + >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False) (1, 2, 3) You should always specify the ``use_list`` keyword argument for backward compatibility. @@ -146,7 +146,7 @@ stream (or from bytes provided through its ``feed`` method). buf.seek(0) - unpacker = msgpack.Unpacker(buf, raw_as_bytes=False) + unpacker = msgpack.Unpacker(buf, raw=False) for unpacked in unpacker: print(unpacked) @@ -179,7 +179,7 @@ It is also possible to pack/unpack custom data types. Here is an example for packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True) - this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw_as_bytes=False) + this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False) ``Unpacker``'s ``object_hook`` callback receives a dict; the ``object_pairs_hook`` callback may instead be used to receive a list of @@ -209,7 +209,7 @@ It is also possible to pack/unpack custom data types using the **ext** type. ... >>> data = array.array('d', [1.2, 3.4]) >>> packed = msgpack.packb(data, default=default, use_bin_type=True) - >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw_as_bytes=False) + >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False) >>> data == unpacked True @@ -257,7 +257,7 @@ For backward compatibility reasons, msgpack-python will still default all strings to byte strings, unless you specify the ``use_bin_type=True`` option in the packer. If you do so, it will use a non-standard type called **bin** to serialize byte arrays, and **raw** becomes to mean **str**. If you want to -distinguish **bin** and **raw** in the unpacker, specify ``raw_as_bytes=False``. +distinguish **bin** and **raw** in the unpacker, specify ``raw=False``. Note that Python 2 defaults to byte-arrays over Unicode strings: @@ -267,7 +267,7 @@ Note that Python 2 defaults to byte-arrays over Unicode strings: >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'])) ['spam', 'eggs'] >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), - raw_as_bytes=False) + raw=False) ['spam', u'eggs'] This is the same code in Python 3 (same behaviour, but Python 3 has a @@ -279,7 +279,7 @@ different default): >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'])) [b'spam', b'eggs'] >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), - raw_as_bytes=False) + raw=False) [b'spam', 'eggs'] diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index b796d04..806be4f 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -43,7 +43,7 @@ from msgpack import ExtType cdef extern from "unpack.h": ctypedef struct msgpack_user: bint use_list - bint raw_as_bytes + bint raw bint has_pairs_hook # call object_hook with k-v pairs PyObject* object_hook PyObject* list_hook @@ -74,14 +74,14 @@ cdef extern from "unpack.h": cdef inline init_ctx(unpack_context *ctx, object object_hook, object object_pairs_hook, object list_hook, object ext_hook, - bint use_list, bint raw_as_bytes, + bint use_list, bint raw, char* encoding, char* unicode_errors, Py_ssize_t max_str_len, Py_ssize_t max_bin_len, Py_ssize_t max_array_len, Py_ssize_t max_map_len, Py_ssize_t max_ext_len): unpack_init(ctx) ctx.user.use_list = use_list - ctx.user.raw_as_bytes = raw_as_bytes + ctx.user.raw = raw ctx.user.object_hook = ctx.user.list_hook = NULL ctx.user.max_str_len = max_str_len ctx.user.max_bin_len = max_bin_len @@ -158,7 +158,7 @@ cdef inline int get_data_from_buffer(object obj, return 1 def unpackb(object packed, object object_hook=None, object list_hook=None, - bint use_list=True, bint raw_as_bytes=True, + bint use_list=True, bint raw=True, encoding=None, unicode_errors="strict", object_pairs_hook=None, ext_hook=ExtType, Py_ssize_t max_str_len=2147483647, # 2**32-1 @@ -185,7 +185,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef int new_protocol = 0 if encoding is not None: - PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw_as_bytes=False instead.", 1) + PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) if isinstance(encoding, unicode): encoding = encoding.encode('ascii') elif not isinstance(encoding, bytes): @@ -203,7 +203,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol) try: init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook, - use_list, raw_as_bytes, cenc, cerr, + use_list, raw, cenc, cerr, max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) ret = unpack_construct(&ctx, buf, buf_len, &off) finally: @@ -261,7 +261,7 @@ cdef class Unpacker(object): If true, unpack msgpack array to Python list. Otherwise, unpack to Python tuple. (default: True) - :param bool raw_as_bytes: + :param bool raw: If true, unpack msgpack raw to Python bytes (default). Otherwise, unpack to Python str (or unicode on Python 2) by decoding with UTF-8 encoding (recommended). @@ -299,7 +299,7 @@ cdef class Unpacker(object): Limits max length of map. (default: 2**31-1) :param str encoding: - Deprecated, use raw_as_bytes instead. + Deprecated, use raw instead. Encoding used for decoding msgpack raw. If it is None (default), msgpack raw is deserialized to Python bytes. @@ -310,13 +310,13 @@ cdef class Unpacker(object): Example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like, raw_as_bytes=False) + unpacker = Unpacker(file_like, raw=False) for o in unpacker: process(o) Example of streaming deserialize from socket:: - unpacker = Unpacker(raw_as_bytes=False) + unpacker = Unpacker(raw=False) while True: buf = sock.recv(1024**2) if not buf: @@ -345,7 +345,7 @@ cdef class Unpacker(object): self.buf = NULL def __init__(self, file_like=None, Py_ssize_t read_size=0, - bint use_list=True, bint raw_as_bytes=True, + bint use_list=True, bint raw=True, object object_hook=None, object object_pairs_hook=None, object list_hook=None, encoding=None, unicode_errors='strict', int max_buffer_size=0, object ext_hook=ExtType, @@ -384,7 +384,7 @@ cdef class Unpacker(object): self.stream_offset = 0 if encoding is not None: - PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw_as_bytes=False instead.", 1) + PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) if isinstance(encoding, unicode): self.encoding = encoding.encode('ascii') elif isinstance(encoding, bytes): @@ -404,7 +404,7 @@ cdef class Unpacker(object): cerr = PyBytes_AsString(self.unicode_errors) init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, - ext_hook, use_list, raw_as_bytes, cenc, cerr, + ext_hook, use_list, raw, cenc, cerr, max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 675ee8a..bf6c9a6 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -145,7 +145,7 @@ class Unpacker(object): If true, unpack msgpack array to Python list. Otherwise, unpack to Python tuple. (default: True) - :param bool raw_as_bytes: + :param bool raw: If true, unpack msgpack raw to Python bytes (default). Otherwise, unpack to Python str (or unicode on Python 2) by decoding with UTF-8 encoding (recommended). @@ -193,13 +193,13 @@ class Unpacker(object): example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like, raw_as_bytes=False) + unpacker = Unpacker(file_like, raw=False) for o in unpacker: process(o) example of streaming deserialize from socket:: - unpacker = Unpacker(raw_as_bytes=False) + unpacker = Unpacker(raw=False) while True: buf = sock.recv(1024**2) if not buf: @@ -209,7 +209,7 @@ class Unpacker(object): process(o) """ - def __init__(self, file_like=None, read_size=0, use_list=True, raw_as_bytes=True, + def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, object_hook=None, object_pairs_hook=None, list_hook=None, encoding=None, unicode_errors=None, max_buffer_size=0, ext_hook=ExtType, @@ -221,7 +221,7 @@ class Unpacker(object): if encoding is not None: warnings.warn( - "encoding is deprecated, Use raw_as_bytes=False instead.", + "encoding is deprecated, Use raw=False instead.", PendingDeprecationWarning) if unicode_errors is not None: @@ -257,7 +257,7 @@ class Unpacker(object): if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") self._read_size = read_size or min(self._max_buffer_size, 16*1024) - self._raw_as_bytes = bool(raw_as_bytes) + self._raw = bool(raw) self._encoding = encoding self._unicode_errors = unicode_errors self._use_list = use_list @@ -606,7 +606,7 @@ class Unpacker(object): if typ == TYPE_RAW: if self._encoding is not None: obj = obj.decode(self._encoding, self._unicode_errors) - elif self._raw_as_bytes: + elif self._raw: obj = bytes(obj) else: obj = obj.decode('utf_8') @@ -715,7 +715,7 @@ class Packer(object): encoding = 'utf_8' else: warnings.warn( - "encoding is deprecated, Use raw_as_bytes=False instead.", + "encoding is deprecated, Use raw=False instead.", PendingDeprecationWarning) if unicode_errors is None: diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 8c2fc46..d7b5e00 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -21,7 +21,7 @@ typedef struct unpack_user { bool use_list; - bool raw_as_bytes; + bool raw; bool has_pairs_hook; PyObject *object_hook; PyObject *list_hook; @@ -229,7 +229,7 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* if (u->encoding) { py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors); - } else if (u->raw_as_bytes) { + } else if (u->raw) { py = PyBytes_FromStringAndSize(p, l); } else { py = PyUnicode_DecodeUTF8(p, l, NULL); diff --git a/test/test_limits.py b/test/test_limits.py index 3febc30..74e48c1 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -39,11 +39,11 @@ def test_max_str_len(): d = 'x' * 3 packed = packb(d) - unpacker = Unpacker(max_str_len=3, raw_as_bytes=False) + unpacker = Unpacker(max_str_len=3, raw=False) unpacker.feed(packed) assert unpacker.unpack() == d - unpacker = Unpacker(max_str_len=2, raw_as_bytes=False) + unpacker = Unpacker(max_str_len=2, raw=False) with pytest.raises(UnpackValueError): unpacker.feed(packed) unpacker.unpack() diff --git a/test/test_pack.py b/test/test_pack.py index 29f5887..b447f9c 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -31,11 +31,11 @@ def testPack(): def testPackUnicode(): test_data = ["", "abcd", ["defgh"], "РуÑÑкий текÑÑ‚"] for td in test_data: - re = unpackb(packb(td), use_list=1, raw_as_bytes=False) + re = unpackb(packb(td), use_list=1, raw=False) assert re == td packer = Packer() data = packer.pack(td) - re = Unpacker(BytesIO(data), raw_as_bytes=False, use_list=1).unpack() + re = Unpacker(BytesIO(data), raw=False, use_list=1).unpack() assert re == td def testPackUTF32(): # deprecated @@ -72,14 +72,14 @@ def testIgnoreUnicodeErrors(): # deprecated def testStrictUnicodeUnpack(): with raises(UnicodeDecodeError): - unpackb(packb(b'abc\xeddef'), raw_as_bytes=False, use_list=1) + unpackb(packb(b'abc\xeddef'), raw=False, use_list=1) def testStrictUnicodePack(): # deprecated with raises(UnicodeEncodeError): packb("abc\xeddef", encoding='ascii', unicode_errors='strict') def testIgnoreErrorsPack(): # deprecated - re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw_as_bytes=False, use_list=1) + re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw=False, use_list=1) assert re == "abcdef" def testDecodeBinary(): diff --git a/test/test_stricttype.py b/test/test_stricttype.py index 13239f1..87e7c1c 100644 --- a/test/test_stricttype.py +++ b/test/test_stricttype.py @@ -11,7 +11,7 @@ def test_namedtuple(): return dict(o._asdict()) raise TypeError('Unsupported type %s' % (type(o),)) packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default) - unpacked = unpackb(packed, raw_as_bytes=False) + unpacked = unpackb(packed, raw=False) assert unpacked == {'foo': 1, 'bar': 42} @@ -32,7 +32,7 @@ def test_tuple(): return o data = packb(t, strict_types=True, use_bin_type=True, default=default) - expected = unpackb(data, raw_as_bytes=False, object_hook=convert) + expected = unpackb(data, raw=False, object_hook=convert) assert expected == t @@ -53,10 +53,10 @@ def test_tuple_ext(): def convert(code, payload): if code == MSGPACK_EXT_TYPE_TUPLE: # Unpack and convert to tuple - return tuple(unpackb(payload, raw_as_bytes=False, ext_hook=convert)) + return tuple(unpackb(payload, raw=False, ext_hook=convert)) raise ValueError('Unknown Ext code {}'.format(code)) data = packb(t, strict_types=True, use_bin_type=True, default=default) - expected = unpackb(data, raw_as_bytes=False, ext_hook=convert) + expected = unpackb(data, raw=False, ext_hook=convert) assert expected == t diff --git a/test/test_unpack.py b/test/test_unpack.py index 143f999..00a1061 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -48,7 +48,7 @@ def test_unpacker_ext_hook(): def __init__(self): super(MyUnpacker, self).__init__( - ext_hook=self._hook, raw_as_bytes=False) + ext_hook=self._hook, raw=False) def _hook(self, code, data): if code == 1: From 52fb85a2c5776590599df3a5839117b88bc49980 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 2 Feb 2018 19:43:42 +0900 Subject: [PATCH 0931/1172] 0.5.2 --- ChangeLog.rst | 13 +++++++++++++ msgpack/_version.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index ed0f92e..ffdc910 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,16 @@ +0.5.2 +====== + +* Add ``raw`` option to Unpacker. It is preferred way than ``encoding`` option. + +* Packer.pack() reset buffer on exception (#274) + + +0.5.1 +====== + +* Remove FutureWarning about use_bin_type option (#271) + 0.5.0 ====== diff --git a/msgpack/_version.py b/msgpack/_version.py index ecba3d8..4e3b29f 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 5, 1) +version = (0, 5, 2) From a0ba076c3527bb474cd8ec820ae0fd6c8293d4af Mon Sep 17 00:00:00 2001 From: Andrew Rabert <6550543+nvllsvm@users.noreply.github.com> Date: Fri, 2 Feb 2018 20:34:42 -0500 Subject: [PATCH 0932/1172] Fix encoding and unicode_errors (#277) Previously, unicode_errors was either set to NULL or to the result of PyBytes_AsString. This restores that behavior while also keeping the existing NULL default behavior. Original defaults were restored to keep API compatibility until these deprecated options are finally removed. --- msgpack/_packer.pyx | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 35e5a9d..c49e719 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -127,27 +127,26 @@ cdef class Packer(object): if not PyCallable_Check(default): raise TypeError("default must be a callable.") self._default = default - if encoding is None: - if unicode_errors is None: - self.encoding = NULL - self.unicode_errors = NULL - else: - self.encoding = "utf_8" - self.unicode_errors = unicode_errors + if encoding is None and unicode_errors is None: + self.encoding = NULL + self.unicode_errors = NULL else: - if isinstance(encoding, unicode): - self._bencoding = encoding.encode('ascii') + if encoding is None: + self.encoding = 'utf-8' else: - self._bencoding = encoding - self.encoding = PyBytes_AsString(self._bencoding) - if isinstance(unicode_errors, unicode): - self._berrors = unicode_errors.encode('ascii') + if isinstance(encoding, unicode): + self._bencoding = encoding.encode('ascii') + else: + self._bencoding = encoding + self.encoding = PyBytes_AsString(self._bencoding) + if unicode_errors is None: + self.unicode_errors = 'strict' else: - self._berrors = unicode_errors - if self._berrors is not None: + if isinstance(unicode_errors, unicode): + self._berrors = unicode_errors.encode('ascii') + else: + self._berrors = unicode_errors self.unicode_errors = PyBytes_AsString(self._berrors) - else: - self.unicode_errors = NULL def __dealloc__(self): PyMem_Free(self.pk.buf) From 618b2cb027e326b315edf5b856ed3b2011c8dad9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 3 Feb 2018 10:54:21 +0900 Subject: [PATCH 0933/1172] 0.5.3 --- ChangeLog.rst | 5 +++++ msgpack/_version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index ffdc910..4a5c599 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,8 @@ +0.5.3 +====== + +* Fixed regression when passing ``unicode_errors`` to Packer but not ``encoding``. (#277) + 0.5.2 ====== diff --git a/msgpack/_version.py b/msgpack/_version.py index 4e3b29f..217bfce 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 5, 2) +version = (0, 5, 3) From 9fdb83719d3544322f0687a9b1637276c1443eda Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 5 Feb 2018 02:19:48 +0900 Subject: [PATCH 0934/1172] Undeprecate unicode_errors (#278) --- msgpack/_packer.pyx | 7 +++---- msgpack/_unpacker.pyx | 9 +++------ msgpack/unpack.h | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index c49e719..b987aee 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -89,10 +89,11 @@ cdef class Packer(object): This is useful when trying to implement accurate serialization for python types. + :param str unicode_errors: + Error handler for encoding unicode. (default: 'strict') + :param str encoding: (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8') - :param str unicode_errors: - (deprecated) Error handler for encoding unicode. (default: 'strict') """ cdef msgpack_packer pk cdef object _default @@ -117,8 +118,6 @@ cdef class Packer(object): bint strict_types=False): if encoding is not None: PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated.", 1) - if unicode_errors is not None: - PyErr_WarnEx(PendingDeprecationWarning, "unicode_errors is deprecated.", 1) self.use_float = use_single_float self.strict_types = strict_types self.autoreset = autoreset diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 806be4f..25fdcd9 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -159,7 +159,7 @@ cdef inline int get_data_from_buffer(object obj, def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=True, bint raw=True, - encoding=None, unicode_errors="strict", + encoding=None, unicode_errors=None, object_pairs_hook=None, ext_hook=ExtType, Py_ssize_t max_str_len=2147483647, # 2**32-1 Py_ssize_t max_bin_len=2147483647, @@ -193,7 +193,6 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cenc = PyBytes_AsString(encoding) if unicode_errors is not None: - PyErr_WarnEx(PendingDeprecationWarning, "unicode_errors is deprecated", 1) if isinstance(unicode_errors, unicode): unicode_errors = unicode_errors.encode('ascii') elif not isinstance(unicode_errors, bytes): @@ -304,8 +303,7 @@ cdef class Unpacker(object): If it is None (default), msgpack raw is deserialized to Python bytes. :param str unicode_errors: - Deprecated. Used for decoding msgpack raw with *encoding*. - (default: `'strict'`) + Error handler used for decoding str type. (default: `'strict'`) Example of streaming deserialize from file-like object:: @@ -347,7 +345,7 @@ cdef class Unpacker(object): def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=True, bint raw=True, object object_hook=None, object object_pairs_hook=None, object list_hook=None, - encoding=None, unicode_errors='strict', int max_buffer_size=0, + encoding=None, unicode_errors=None, int max_buffer_size=0, object ext_hook=ExtType, Py_ssize_t max_str_len=2147483647, # 2**32-1 Py_ssize_t max_bin_len=2147483647, @@ -394,7 +392,6 @@ cdef class Unpacker(object): cenc = PyBytes_AsString(self.encoding) if unicode_errors is not None: - PyErr_WarnEx(PendingDeprecationWarning, "unicode_errors is deprecated", 1) if isinstance(unicode_errors, unicode): self.unicode_errors = unicode_errors.encode('ascii') elif isinstance(unicode_errors, bytes): diff --git a/msgpack/unpack.h b/msgpack/unpack.h index d7b5e00..63e5543 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -232,7 +232,7 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* } else if (u->raw) { py = PyBytes_FromStringAndSize(p, l); } else { - py = PyUnicode_DecodeUTF8(p, l, NULL); + py = PyUnicode_DecodeUTF8(p, l, u->unicode_errors); } if (!py) return -1; From 351023946fa92fc78bdd5247f64d3a62fa233e95 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 5 Feb 2018 02:25:12 +0900 Subject: [PATCH 0935/1172] 0.5.4 --- ChangeLog.rst | 5 +++++ msgpack/_version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 4a5c599..67ee24c 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,8 @@ +0.5.4 +====== + +* Undeprecate ``unicode_errors`` option. (#278) + 0.5.3 ====== diff --git a/msgpack/_version.py b/msgpack/_version.py index 217bfce..e7435d9 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 5, 3) +version = (0, 5, 4) From 2644cbdcb7e1f2a8373de5bd30d71c907827caf6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 5 Feb 2018 11:44:17 +0900 Subject: [PATCH 0936/1172] Use cython's cast for converting encoding and errors (#279) It is little faster on Python 3 because we can skip temporary bytes object --- msgpack/_packer.pyx | 40 +++++++++++++++++--------------------- msgpack/_unpacker.pyx | 45 ++++++++++++++----------------------------- 2 files changed, 32 insertions(+), 53 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index b987aee..225f24a 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -1,7 +1,8 @@ # coding: utf-8 -#cython: embedsignature=True +#cython: embedsignature=True, c_string_encoding=ascii from cpython cimport * +from cpython.version cimport PY_MAJOR_VERSION from cpython.exc cimport PyErr_WarnEx from msgpack.exceptions import PackValueError, PackOverflowError @@ -99,8 +100,8 @@ cdef class Packer(object): cdef object _default cdef object _bencoding cdef object _berrors - cdef char *encoding - cdef char *unicode_errors + cdef const char *encoding + cdef const char *unicode_errors cdef bint strict_types cdef bool use_float cdef bint autoreset @@ -126,26 +127,21 @@ cdef class Packer(object): if not PyCallable_Check(default): raise TypeError("default must be a callable.") self._default = default - if encoding is None and unicode_errors is None: - self.encoding = NULL - self.unicode_errors = NULL - else: - if encoding is None: + + self._bencoding = encoding + if encoding is None: + if PY_MAJOR_VERSION < 3: self.encoding = 'utf-8' else: - if isinstance(encoding, unicode): - self._bencoding = encoding.encode('ascii') - else: - self._bencoding = encoding - self.encoding = PyBytes_AsString(self._bencoding) - if unicode_errors is None: - self.unicode_errors = 'strict' - else: - if isinstance(unicode_errors, unicode): - self._berrors = unicode_errors.encode('ascii') - else: - self._berrors = unicode_errors - self.unicode_errors = PyBytes_AsString(self._berrors) + self.encoding = NULL + else: + self.encoding = self._bencoding + + self._berrors = unicode_errors + if unicode_errors is None: + self.unicode_errors = NULL + else: + self.unicode_errors = self._berrors def __dealloc__(self): PyMem_Free(self.pk.buf) @@ -212,7 +208,7 @@ cdef class Packer(object): if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, L) elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): - if self.encoding == NULL: + if self.encoding == NULL and self.unicode_errors == NULL: ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); if ret == -2: raise PackValueError("unicode string is too large") diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 25fdcd9..fb58490 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -1,6 +1,7 @@ # coding: utf-8 -#cython: embedsignature=True +#cython: embedsignature=True, c_string_encoding=ascii +from cpython.version cimport PY_MAJOR_VERSION from cpython.bytes cimport ( PyBytes_AsString, PyBytes_FromStringAndSize, @@ -75,7 +76,7 @@ cdef inline init_ctx(unpack_context *ctx, object object_hook, object object_pairs_hook, object list_hook, object ext_hook, bint use_list, bint raw, - char* encoding, char* unicode_errors, + const char* encoding, const char* unicode_errors, Py_ssize_t max_str_len, Py_ssize_t max_bin_len, Py_ssize_t max_array_len, Py_ssize_t max_map_len, Py_ssize_t max_ext_len): @@ -180,24 +181,16 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef Py_buffer view cdef char* buf = NULL cdef Py_ssize_t buf_len - cdef char* cenc = NULL - cdef char* cerr = NULL + cdef const char* cenc = NULL + cdef const char* cerr = NULL cdef int new_protocol = 0 if encoding is not None: PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) - if isinstance(encoding, unicode): - encoding = encoding.encode('ascii') - elif not isinstance(encoding, bytes): - raise TypeError("encoding should be bytes or unicode") - cenc = PyBytes_AsString(encoding) + cenc = encoding if unicode_errors is not None: - if isinstance(unicode_errors, unicode): - unicode_errors = unicode_errors.encode('ascii') - elif not isinstance(unicode_errors, bytes): - raise TypeError("unicode_errors should be bytes or unicode") - cerr = PyBytes_AsString(unicode_errors) + cerr = unicode_errors get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol) try: @@ -219,7 +212,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, def unpack(object stream, object object_hook=None, object list_hook=None, - bint use_list=1, encoding=None, unicode_errors="strict", + bint use_list=1, encoding=None, unicode_errors=None, object_pairs_hook=None, ext_hook=ExtType, Py_ssize_t max_str_len=2147483647, # 2**32-1 Py_ssize_t max_bin_len=2147483647, @@ -352,8 +345,8 @@ cdef class Unpacker(object): Py_ssize_t max_array_len=2147483647, Py_ssize_t max_map_len=2147483647, Py_ssize_t max_ext_len=2147483647): - cdef char *cenc=NULL, - cdef char *cerr=NULL + cdef const char *cenc=NULL, + cdef const char *cerr=NULL self.object_hook = object_hook self.object_pairs_hook = object_pairs_hook @@ -383,22 +376,12 @@ cdef class Unpacker(object): if encoding is not None: PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) - if isinstance(encoding, unicode): - self.encoding = encoding.encode('ascii') - elif isinstance(encoding, bytes): - self.encoding = encoding - else: - raise TypeError("encoding should be bytes or unicode") - cenc = PyBytes_AsString(self.encoding) + self.encoding = encoding + cenc = encoding if unicode_errors is not None: - if isinstance(unicode_errors, unicode): - self.unicode_errors = unicode_errors.encode('ascii') - elif isinstance(unicode_errors, bytes): - self.unicode_errors = unicode_errors - else: - raise TypeError("unicode_errors should be bytes or unicode") - cerr = PyBytes_AsString(self.unicode_errors) + self.unicode_errors = unicode_errors + cerr = unicode_errors init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, ext_hook, use_list, raw, cenc, cerr, From 4b72b6177321ef24b6c7af2354fb980db69e2aec Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 5 Feb 2018 15:08:19 +0900 Subject: [PATCH 0937/1172] Add Makefile target for updating docker image --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 4030080..124f243 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,11 @@ clean: rm -rf msgpack/__pycache__ rm -rf test/__pycache__ +.PHONY: update-docker +update-docker: + docker pull quay.io/pypa/manylinux1_i686 + docker pull quay.io/pypa/manylinux1_x86_64 + .PHONY: linux-wheel linux-wheel: docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_i686 bash docker/buildwheel.sh From ae8d4694829d5b58d613c588c30e29dd29860c4f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 16 Feb 2018 16:35:22 +0900 Subject: [PATCH 0938/1172] Fix memory leak in pure Python Unpacker.feed() (#284) fixes #283 --- msgpack/fallback.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index bf6c9a6..7e40686 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -289,6 +289,8 @@ class Unpacker(object): view = _get_data_from_buffer(next_bytes) if (len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size): raise BufferFull + del self._buffer[:self._buff_i] + self._buff_i = 0 self._buffer += view def _consume(self): From da902f9c1d996fb461f1efef6487ef40d32d365a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 22 Feb 2018 00:55:32 +0900 Subject: [PATCH 0939/1172] Move unpack() from each implementation to __init__. (#286) Fixes #285 --- msgpack/__init__.py | 17 ++++++++++++++--- msgpack/_unpacker.pyx | 26 -------------------------- msgpack/fallback.py | 10 ---------- 3 files changed, 14 insertions(+), 39 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 6c5ae53..70de97f 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -19,13 +19,13 @@ class ExtType(namedtuple('ExtType', 'code data')): import os if os.environ.get('MSGPACK_PUREPYTHON'): - from msgpack.fallback import Packer, unpack, unpackb, Unpacker + from msgpack.fallback import Packer, unpackb, Unpacker else: try: from msgpack._packer import Packer - from msgpack._unpacker import unpack, unpackb, Unpacker + from msgpack._unpacker import unpackb, Unpacker except ImportError: - from msgpack.fallback import Packer, unpack, unpackb, Unpacker + from msgpack.fallback import Packer, unpackb, Unpacker def pack(o, stream, **kwargs): @@ -46,6 +46,17 @@ def packb(o, **kwargs): """ return Packer(**kwargs).pack(o) + +def unpack(stream, **kwargs): + """ + Unpack an object from `stream`. + + Raises `ExtraData` when `packed` contains extra bytes. + See :class:`Unpacker` for options. + """ + return unpackb(stream.read(), **kwargs) + + # alias for compatibility to simplejson/marshal/pickle. load = unpack loads = unpackb diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index fb58490..25a7401 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -211,32 +211,6 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, raise UnpackValueError("Unpack failed: error = %d" % (ret,)) -def unpack(object stream, object object_hook=None, object list_hook=None, - bint use_list=1, encoding=None, unicode_errors=None, - object_pairs_hook=None, ext_hook=ExtType, - Py_ssize_t max_str_len=2147483647, # 2**32-1 - Py_ssize_t max_bin_len=2147483647, - Py_ssize_t max_array_len=2147483647, - Py_ssize_t max_map_len=2147483647, - Py_ssize_t max_ext_len=2147483647): - """ - Unpack an object from `stream`. - - Raises `ValueError` when `stream` has extra bytes. - - See :class:`Unpacker` for options. - """ - return unpackb(stream.read(), use_list=use_list, - object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, - encoding=encoding, unicode_errors=unicode_errors, ext_hook=ext_hook, - max_str_len=max_str_len, - max_bin_len=max_bin_len, - max_array_len=max_array_len, - max_map_len=max_map_len, - max_ext_len=max_ext_len, - ) - - cdef class Unpacker(object): """Streaming unpacker. diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 7e40686..3609fd8 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -100,16 +100,6 @@ def _get_data_from_buffer(obj): return view -def unpack(stream, **kwargs): - """ - Unpack an object from `stream`. - - Raises `ExtraData` when `packed` contains extra bytes. - See :class:`Unpacker` for options. - """ - data = stream.read() - return unpackb(data, **kwargs) - def unpackb(packed, **kwargs): """ From 02c881c7cb402b37418a9bd9a3fa56daf673a71b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 22 Feb 2018 17:55:39 +0900 Subject: [PATCH 0940/1172] 0.5.5 --- ChangeLog.rst | 6 ++++++ msgpack/_version.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 67ee24c..263d1f3 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,9 @@ +0.5.5 +====== + +* Fix memory leak in pure Python Unpacker.feed() (#283) +* Fix unpack() didn't support `raw` option (#285) + 0.5.4 ====== diff --git a/msgpack/_version.py b/msgpack/_version.py index e7435d9..4035588 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 5, 4) +version = (0, 5, 5) From 9bf38105f7dfd7e9885d8faee81c8bd188b4de4d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 23 Feb 2018 11:32:26 +0900 Subject: [PATCH 0941/1172] Revert "0.5.5" This reverts commit 02c881c7cb402b37418a9bd9a3fa56daf673a71b. From 9455fccc5283abe59868c55ee3f4cedd5bf2d14b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 23 Feb 2018 11:32:26 +0900 Subject: [PATCH 0942/1172] Revert "Move unpack() from each implementation to __init__. (#286)" This reverts commit da902f9c1d996fb461f1efef6487ef40d32d365a. --- msgpack/__init__.py | 17 +++-------------- msgpack/_unpacker.pyx | 26 ++++++++++++++++++++++++++ msgpack/fallback.py | 10 ++++++++++ 3 files changed, 39 insertions(+), 14 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 70de97f..6c5ae53 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -19,13 +19,13 @@ class ExtType(namedtuple('ExtType', 'code data')): import os if os.environ.get('MSGPACK_PUREPYTHON'): - from msgpack.fallback import Packer, unpackb, Unpacker + from msgpack.fallback import Packer, unpack, unpackb, Unpacker else: try: from msgpack._packer import Packer - from msgpack._unpacker import unpackb, Unpacker + from msgpack._unpacker import unpack, unpackb, Unpacker except ImportError: - from msgpack.fallback import Packer, unpackb, Unpacker + from msgpack.fallback import Packer, unpack, unpackb, Unpacker def pack(o, stream, **kwargs): @@ -46,17 +46,6 @@ def packb(o, **kwargs): """ return Packer(**kwargs).pack(o) - -def unpack(stream, **kwargs): - """ - Unpack an object from `stream`. - - Raises `ExtraData` when `packed` contains extra bytes. - See :class:`Unpacker` for options. - """ - return unpackb(stream.read(), **kwargs) - - # alias for compatibility to simplejson/marshal/pickle. load = unpack loads = unpackb diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 25a7401..fb58490 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -211,6 +211,32 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, raise UnpackValueError("Unpack failed: error = %d" % (ret,)) +def unpack(object stream, object object_hook=None, object list_hook=None, + bint use_list=1, encoding=None, unicode_errors=None, + object_pairs_hook=None, ext_hook=ExtType, + Py_ssize_t max_str_len=2147483647, # 2**32-1 + Py_ssize_t max_bin_len=2147483647, + Py_ssize_t max_array_len=2147483647, + Py_ssize_t max_map_len=2147483647, + Py_ssize_t max_ext_len=2147483647): + """ + Unpack an object from `stream`. + + Raises `ValueError` when `stream` has extra bytes. + + See :class:`Unpacker` for options. + """ + return unpackb(stream.read(), use_list=use_list, + object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, + encoding=encoding, unicode_errors=unicode_errors, ext_hook=ext_hook, + max_str_len=max_str_len, + max_bin_len=max_bin_len, + max_array_len=max_array_len, + max_map_len=max_map_len, + max_ext_len=max_ext_len, + ) + + cdef class Unpacker(object): """Streaming unpacker. diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 3609fd8..7e40686 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -100,6 +100,16 @@ def _get_data_from_buffer(obj): return view +def unpack(stream, **kwargs): + """ + Unpack an object from `stream`. + + Raises `ExtraData` when `packed` contains extra bytes. + See :class:`Unpacker` for options. + """ + data = stream.read() + return unpackb(data, **kwargs) + def unpackb(packed, **kwargs): """ From fbaa1360be67672f855dd6337e4137f2edf7ade6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 23 Feb 2018 11:35:09 +0900 Subject: [PATCH 0943/1172] Fix #285 again --- msgpack/_unpacker.pyx | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index fb58490..3843e92 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -211,30 +211,15 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, raise UnpackValueError("Unpack failed: error = %d" % (ret,)) -def unpack(object stream, object object_hook=None, object list_hook=None, - bint use_list=1, encoding=None, unicode_errors=None, - object_pairs_hook=None, ext_hook=ExtType, - Py_ssize_t max_str_len=2147483647, # 2**32-1 - Py_ssize_t max_bin_len=2147483647, - Py_ssize_t max_array_len=2147483647, - Py_ssize_t max_map_len=2147483647, - Py_ssize_t max_ext_len=2147483647): +def unpack(object stream, **kwargs): """ Unpack an object from `stream`. Raises `ValueError` when `stream` has extra bytes. - See :class:`Unpacker` for options. """ - return unpackb(stream.read(), use_list=use_list, - object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, - encoding=encoding, unicode_errors=unicode_errors, ext_hook=ext_hook, - max_str_len=max_str_len, - max_bin_len=max_bin_len, - max_array_len=max_array_len, - max_map_len=max_map_len, - max_ext_len=max_ext_len, - ) + data = stream.read() + return unpackb(data, **kwargs) cdef class Unpacker(object): From f38c1a3674b7623080cb774b56ede21383bde04a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 23 Feb 2018 11:52:48 +0900 Subject: [PATCH 0944/1172] Fix Unpacker.feed() drops unused data in buffer. (#289) Fixes #287 --- msgpack/fallback.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 7e40686..e9108d2 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -289,8 +289,13 @@ class Unpacker(object): view = _get_data_from_buffer(next_bytes) if (len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size): raise BufferFull - del self._buffer[:self._buff_i] - self._buff_i = 0 + + # Strip buffer before checkpoint before reading file. + if self._buf_checkpoint > 0: + del self._buffer[:self._buf_checkpoint] + self._buff_i -= self._buf_checkpoint + self._buf_checkpoint = 0 + self._buffer += view def _consume(self): From ae3a6ba0b04487d7108360d1d504332aed079556 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 23 Feb 2018 15:41:21 +0900 Subject: [PATCH 0945/1172] Deprecate implementation module's unpack() (#290) --- msgpack/__init__.py | 18 +++++++++++++++--- msgpack/_unpacker.pyx | 9 +++------ msgpack/fallback.py | 21 +++++---------------- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 6c5ae53..3955a41 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -19,13 +19,13 @@ class ExtType(namedtuple('ExtType', 'code data')): import os if os.environ.get('MSGPACK_PUREPYTHON'): - from msgpack.fallback import Packer, unpack, unpackb, Unpacker + from msgpack.fallback import Packer, unpackb, Unpacker else: try: from msgpack._packer import Packer - from msgpack._unpacker import unpack, unpackb, Unpacker + from msgpack._unpacker import unpackb, Unpacker except ImportError: - from msgpack.fallback import Packer, unpack, unpackb, Unpacker + from msgpack.fallback import Packer, unpackb, Unpacker def pack(o, stream, **kwargs): @@ -46,6 +46,18 @@ def packb(o, **kwargs): """ return Packer(**kwargs).pack(o) + +def unpack(stream, **kwargs): + """ + Unpack an object from `stream`. + + Raises `ExtraData` when `stream` contains extra bytes. + See :class:`Unpacker` for options. + """ + data = stream.read() + return unpackb(data, **kwargs) + + # alias for compatibility to simplejson/marshal/pickle. load = unpack loads = unpackb diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 3843e92..d7fa5bc 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -212,12 +212,9 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, def unpack(object stream, **kwargs): - """ - Unpack an object from `stream`. - - Raises `ValueError` when `stream` has extra bytes. - See :class:`Unpacker` for options. - """ + PyErr_WarnEx( + PendingDeprecationWarning, + "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", 1) data = stream.read() return unpackb(data, **kwargs) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index e9108d2..c0e5fd6 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -101,12 +101,9 @@ def _get_data_from_buffer(obj): def unpack(stream, **kwargs): - """ - Unpack an object from `stream`. - - Raises `ExtraData` when `packed` contains extra bytes. - See :class:`Unpacker` for options. - """ + warnings.warn( + "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", + PendingDeprecationWarning) data = stream.read() return unpackb(data, **kwargs) @@ -224,11 +221,7 @@ class Unpacker(object): "encoding is deprecated, Use raw=False instead.", PendingDeprecationWarning) - if unicode_errors is not None: - warnings.warn( - "unicode_errors is deprecated.", - PendingDeprecationWarning) - else: + if unicode_errors is None: unicode_errors = 'strict' if file_like is None: @@ -713,7 +706,7 @@ class Packer(object): (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8') :param str unicode_errors: - (deprecated) Error handler for encoding unicode. (default: 'strict') + Error handler for encoding unicode. (default: 'strict') """ def __init__(self, default=None, encoding=None, unicode_errors=None, use_single_float=False, autoreset=True, use_bin_type=False, @@ -727,10 +720,6 @@ class Packer(object): if unicode_errors is None: unicode_errors = 'strict' - else: - warnings.warn( - "unicode_errors is deprecated.", - PendingDeprecationWarning) self._strict_types = strict_types self._use_float = use_single_float From d4675bee6c83d42cecda9a84e6716021eb2ad679 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 23 Feb 2018 15:45:34 +0900 Subject: [PATCH 0946/1172] 0.5.6 --- ChangeLog.rst | 7 +++++++ msgpack/_version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 263d1f3..b6158c3 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,10 @@ +0.5.6 +====== + +* Fix fallback.Unpacker.feed() dropped unused data from buffer (#287) +* Resurrect fallback.unpack() and _unpacker.unpack(). + They were removed at 0.5.5 but it breaks backward compatibility. (#288, #290) + 0.5.5 ====== diff --git a/msgpack/_version.py b/msgpack/_version.py index 4035588..d28f0de 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 5, 5) +version = (0, 5, 6) From 984116bd1805c52fb4e7c07bfd9635ed902850c3 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 13 Apr 2018 23:41:01 +0900 Subject: [PATCH 0947/1172] Update setup() --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6108587..c252d81 100755 --- a/setup.py +++ b/setup.py @@ -120,7 +120,13 @@ setup(name=name, packages=['msgpack'], description=desc, long_description=long_desc, - url='http://msgpack.org/', + long_description_content_type="text/x-rst", + url='https://msgpack.org/', + project_urls = { + 'Documentation': 'https://msgpack-python.readthedocs.io/', + 'Source': 'https://github.com/msgpack/msgpack-python', + 'Tracker': 'https://github.com/msgpack/msgpack-python/issues', + }, license='Apache 2.0', classifiers=[ 'Programming Language :: Python :: 2', From b10cf78f54a5daab866b19c32e45e207d838f52b Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 15 Apr 2018 23:18:35 -0400 Subject: [PATCH 0948/1172] Fix TypeError in fallback.unpack() on H", self._buffer, self._buff_i)[0] + n = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0] self._buff_i += 2 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) @@ -396,7 +402,7 @@ class Unpacker(object): elif b == 0xc6: typ = TYPE_BIN self._reserve(4) - n = struct.unpack_from(">I", self._buffer, self._buff_i)[0] + n = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0] self._buff_i += 4 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) @@ -404,7 +410,7 @@ class Unpacker(object): elif b == 0xc7: # ext 8 typ = TYPE_EXT self._reserve(2) - L, n = struct.unpack_from('Bb', self._buffer, self._buff_i) + L, n = struct.unpack_from('Bb', self._buffer_view, self._buff_i) self._buff_i += 2 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) @@ -412,7 +418,7 @@ class Unpacker(object): elif b == 0xc8: # ext 16 typ = TYPE_EXT self._reserve(3) - L, n = struct.unpack_from('>Hb', self._buffer, self._buff_i) + L, n = struct.unpack_from('>Hb', self._buffer_view, self._buff_i) self._buff_i += 3 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) @@ -420,18 +426,18 @@ class Unpacker(object): elif b == 0xc9: # ext 32 typ = TYPE_EXT self._reserve(5) - L, n = struct.unpack_from('>Ib', self._buffer, self._buff_i) + L, n = struct.unpack_from('>Ib', self._buffer_view, self._buff_i) self._buff_i += 5 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) elif b == 0xca: self._reserve(4) - obj = struct.unpack_from(">f", self._buffer, self._buff_i)[0] + obj = struct.unpack_from(">f", self._buffer_view, self._buff_i)[0] self._buff_i += 4 elif b == 0xcb: self._reserve(8) - obj = struct.unpack_from(">d", self._buffer, self._buff_i)[0] + obj = struct.unpack_from(">d", self._buffer_view, self._buff_i)[0] self._buff_i += 8 elif b == 0xcc: self._reserve(1) @@ -439,66 +445,66 @@ class Unpacker(object): self._buff_i += 1 elif b == 0xcd: self._reserve(2) - obj = struct.unpack_from(">H", self._buffer, self._buff_i)[0] + obj = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0] self._buff_i += 2 elif b == 0xce: self._reserve(4) - obj = struct.unpack_from(">I", self._buffer, self._buff_i)[0] + obj = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0] self._buff_i += 4 elif b == 0xcf: self._reserve(8) - obj = struct.unpack_from(">Q", self._buffer, self._buff_i)[0] + obj = struct.unpack_from(">Q", self._buffer_view, self._buff_i)[0] self._buff_i += 8 elif b == 0xd0: self._reserve(1) - obj = struct.unpack_from("b", self._buffer, self._buff_i)[0] + obj = struct.unpack_from("b", self._buffer_view, self._buff_i)[0] self._buff_i += 1 elif b == 0xd1: self._reserve(2) - obj = struct.unpack_from(">h", self._buffer, self._buff_i)[0] + obj = struct.unpack_from(">h", self._buffer_view, self._buff_i)[0] self._buff_i += 2 elif b == 0xd2: self._reserve(4) - obj = struct.unpack_from(">i", self._buffer, self._buff_i)[0] + obj = struct.unpack_from(">i", self._buffer_view, self._buff_i)[0] self._buff_i += 4 elif b == 0xd3: self._reserve(8) - obj = struct.unpack_from(">q", self._buffer, self._buff_i)[0] + obj = struct.unpack_from(">q", self._buffer_view, self._buff_i)[0] self._buff_i += 8 elif b == 0xd4: # fixext 1 typ = TYPE_EXT if self._max_ext_len < 1: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) self._reserve(2) - n, obj = struct.unpack_from("b1s", self._buffer, self._buff_i) + n, obj = struct.unpack_from("b1s", self._buffer_view, self._buff_i) self._buff_i += 2 elif b == 0xd5: # fixext 2 typ = TYPE_EXT if self._max_ext_len < 2: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) self._reserve(3) - n, obj = struct.unpack_from("b2s", self._buffer, self._buff_i) + n, obj = struct.unpack_from("b2s", self._buffer_view, self._buff_i) self._buff_i += 3 elif b == 0xd6: # fixext 4 typ = TYPE_EXT if self._max_ext_len < 4: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) self._reserve(5) - n, obj = struct.unpack_from("b4s", self._buffer, self._buff_i) + n, obj = struct.unpack_from("b4s", self._buffer_view, self._buff_i) self._buff_i += 5 elif b == 0xd7: # fixext 8 typ = TYPE_EXT if self._max_ext_len < 8: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) self._reserve(9) - n, obj = struct.unpack_from("b8s", self._buffer, self._buff_i) + n, obj = struct.unpack_from("b8s", self._buffer_view, self._buff_i) self._buff_i += 9 elif b == 0xd8: # fixext 16 typ = TYPE_EXT if self._max_ext_len < 16: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) self._reserve(17) - n, obj = struct.unpack_from("b16s", self._buffer, self._buff_i) + n, obj = struct.unpack_from("b16s", self._buffer_view, self._buff_i) self._buff_i += 17 elif b == 0xd9: typ = TYPE_RAW @@ -511,7 +517,7 @@ class Unpacker(object): elif b == 0xda: typ = TYPE_RAW self._reserve(2) - n, = struct.unpack_from(">H", self._buffer, self._buff_i) + n, = struct.unpack_from(">H", self._buffer_view, self._buff_i) self._buff_i += 2 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) @@ -519,7 +525,7 @@ class Unpacker(object): elif b == 0xdb: typ = TYPE_RAW self._reserve(4) - n, = struct.unpack_from(">I", self._buffer, self._buff_i) + n, = struct.unpack_from(">I", self._buffer_view, self._buff_i) self._buff_i += 4 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) @@ -527,27 +533,27 @@ class Unpacker(object): elif b == 0xdc: typ = TYPE_ARRAY self._reserve(2) - n, = struct.unpack_from(">H", self._buffer, self._buff_i) + n, = struct.unpack_from(">H", self._buffer_view, self._buff_i) self._buff_i += 2 if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xdd: typ = TYPE_ARRAY self._reserve(4) - n, = struct.unpack_from(">I", self._buffer, self._buff_i) + n, = struct.unpack_from(">I", self._buffer_view, self._buff_i) self._buff_i += 4 if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xde: self._reserve(2) - n, = struct.unpack_from(">H", self._buffer, self._buff_i) + n, = struct.unpack_from(">H", self._buffer_view, self._buff_i) self._buff_i += 2 if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP elif b == 0xdf: self._reserve(4) - n, = struct.unpack_from(">I", self._buffer, self._buff_i) + n, = struct.unpack_from(">I", self._buffer_view, self._buff_i) self._buff_i += 4 if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) From 5f684aed82d0d08079b9aa74e1d41cc2a367833d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 27 Jun 2018 01:27:31 +0900 Subject: [PATCH 0949/1172] fallback: Fix error on Jython Fixes #303 --- msgpack/fallback.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index c8c8c78..20ad4c9 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -100,6 +100,13 @@ def _get_data_from_buffer(obj): return view +# Jython's memoryview support is incomplete +# See https://github.com/msgpack/msgpack-python/issues/303 +_is_jython = sys.platform.startswith('java') +if _is_jython: + _get_data_from_buffer = bytes + + def unpack(stream, **kwargs): warnings.warn( "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", From d1060de29309717355b78fe9092f7995897b4f0c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 13 Jul 2018 19:54:44 +0900 Subject: [PATCH 0950/1172] travis: Run test on Python 3.4 (#307) Python 3.4 is not supported officially. But keep running test for a while, to know when msgpack-python stop working on Python 3.4 actually. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 7aac664..823c8b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ cache: pip python: - "2.7" + - "3.4" - "3.5" - "3.6" - "3.7-dev" From 70b5f21b34b28cd50125ee337b5a399001759b73 Mon Sep 17 00:00:00 2001 From: Raymond E Ferguson Date: Tue, 2 Oct 2018 11:20:06 +0000 Subject: [PATCH 0951/1172] Alternate fixes for jython and legacy CPython (#310) Python 3.4 is not supported officially. But keep running test for a while, to know when msgpack-python stop working on Python 3.4 actually. The current patches did not work under jython-2.7.1 where implicit casting of buffer or memoryview doesn't work. It may also be the jython is a little pickier about string casting non string bytes due to the underlying strong typing of java. See issues #303 & #304. --- msgpack/fallback.py | 74 +++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 39 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 20ad4c9..0b60ba2 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -100,13 +100,6 @@ def _get_data_from_buffer(obj): return view -# Jython's memoryview support is incomplete -# See https://github.com/msgpack/msgpack-python/issues/303 -_is_jython = sys.platform.startswith('java') -if _is_jython: - _get_data_from_buffer = bytes - - def unpack(stream, **kwargs): warnings.warn( "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", @@ -133,6 +126,14 @@ def unpackb(packed, **kwargs): return ret +if sys.version_info < (2, 7, 6): + def _unpack_from(f, b, o=0): + """Explicit typcast for legacy struct.unpack_from""" + return struct.unpack_from(f, bytes(b), o) +else: + _unpack_from = struct.unpack_from + + class Unpacker(object): """Streaming unpacker. @@ -241,12 +242,6 @@ class Unpacker(object): #: array of bytes fed. self._buffer = bytearray() - # Some very old pythons don't support `struct.unpack_from()` with a - # `bytearray`. So we wrap it in a `buffer()` there. - if sys.version_info < (2, 7, 6): - self._buffer_view = buffer(self._buffer) - else: - self._buffer_view = self._buffer #: Which position we currently reads self._buff_i = 0 @@ -302,7 +297,8 @@ class Unpacker(object): self._buff_i -= self._buf_checkpoint self._buf_checkpoint = 0 - self._buffer += view + # Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython + self._buffer.extend(view) def _consume(self): """ Gets rid of the used parts of the buffer. """ @@ -401,7 +397,7 @@ class Unpacker(object): elif b == 0xc5: typ = TYPE_BIN self._reserve(2) - n = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0] + n = _unpack_from(">H", self._buffer, self._buff_i)[0] self._buff_i += 2 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) @@ -409,7 +405,7 @@ class Unpacker(object): elif b == 0xc6: typ = TYPE_BIN self._reserve(4) - n = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0] + n = _unpack_from(">I", self._buffer, self._buff_i)[0] self._buff_i += 4 if n > self._max_bin_len: raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) @@ -417,7 +413,7 @@ class Unpacker(object): elif b == 0xc7: # ext 8 typ = TYPE_EXT self._reserve(2) - L, n = struct.unpack_from('Bb', self._buffer_view, self._buff_i) + L, n = _unpack_from('Bb', self._buffer, self._buff_i) self._buff_i += 2 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) @@ -425,7 +421,7 @@ class Unpacker(object): elif b == 0xc8: # ext 16 typ = TYPE_EXT self._reserve(3) - L, n = struct.unpack_from('>Hb', self._buffer_view, self._buff_i) + L, n = _unpack_from('>Hb', self._buffer, self._buff_i) self._buff_i += 3 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) @@ -433,18 +429,18 @@ class Unpacker(object): elif b == 0xc9: # ext 32 typ = TYPE_EXT self._reserve(5) - L, n = struct.unpack_from('>Ib', self._buffer_view, self._buff_i) + L, n = _unpack_from('>Ib', self._buffer, self._buff_i) self._buff_i += 5 if L > self._max_ext_len: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) elif b == 0xca: self._reserve(4) - obj = struct.unpack_from(">f", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">f", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xcb: self._reserve(8) - obj = struct.unpack_from(">d", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">d", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xcc: self._reserve(1) @@ -452,66 +448,66 @@ class Unpacker(object): self._buff_i += 1 elif b == 0xcd: self._reserve(2) - obj = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">H", self._buffer, self._buff_i)[0] self._buff_i += 2 elif b == 0xce: self._reserve(4) - obj = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">I", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xcf: self._reserve(8) - obj = struct.unpack_from(">Q", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">Q", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xd0: self._reserve(1) - obj = struct.unpack_from("b", self._buffer_view, self._buff_i)[0] + obj = _unpack_from("b", self._buffer, self._buff_i)[0] self._buff_i += 1 elif b == 0xd1: self._reserve(2) - obj = struct.unpack_from(">h", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">h", self._buffer, self._buff_i)[0] self._buff_i += 2 elif b == 0xd2: self._reserve(4) - obj = struct.unpack_from(">i", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">i", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xd3: self._reserve(8) - obj = struct.unpack_from(">q", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">q", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xd4: # fixext 1 typ = TYPE_EXT if self._max_ext_len < 1: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) self._reserve(2) - n, obj = struct.unpack_from("b1s", self._buffer_view, self._buff_i) + n, obj = _unpack_from("b1s", self._buffer, self._buff_i) self._buff_i += 2 elif b == 0xd5: # fixext 2 typ = TYPE_EXT if self._max_ext_len < 2: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) self._reserve(3) - n, obj = struct.unpack_from("b2s", self._buffer_view, self._buff_i) + n, obj = _unpack_from("b2s", self._buffer, self._buff_i) self._buff_i += 3 elif b == 0xd6: # fixext 4 typ = TYPE_EXT if self._max_ext_len < 4: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) self._reserve(5) - n, obj = struct.unpack_from("b4s", self._buffer_view, self._buff_i) + n, obj = _unpack_from("b4s", self._buffer, self._buff_i) self._buff_i += 5 elif b == 0xd7: # fixext 8 typ = TYPE_EXT if self._max_ext_len < 8: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) self._reserve(9) - n, obj = struct.unpack_from("b8s", self._buffer_view, self._buff_i) + n, obj = _unpack_from("b8s", self._buffer, self._buff_i) self._buff_i += 9 elif b == 0xd8: # fixext 16 typ = TYPE_EXT if self._max_ext_len < 16: raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) self._reserve(17) - n, obj = struct.unpack_from("b16s", self._buffer_view, self._buff_i) + n, obj = _unpack_from("b16s", self._buffer, self._buff_i) self._buff_i += 17 elif b == 0xd9: typ = TYPE_RAW @@ -524,7 +520,7 @@ class Unpacker(object): elif b == 0xda: typ = TYPE_RAW self._reserve(2) - n, = struct.unpack_from(">H", self._buffer_view, self._buff_i) + n, = _unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) @@ -532,7 +528,7 @@ class Unpacker(object): elif b == 0xdb: typ = TYPE_RAW self._reserve(4) - n, = struct.unpack_from(">I", self._buffer_view, self._buff_i) + n, = _unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_str_len: raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) @@ -540,27 +536,27 @@ class Unpacker(object): elif b == 0xdc: typ = TYPE_ARRAY self._reserve(2) - n, = struct.unpack_from(">H", self._buffer_view, self._buff_i) + n, = _unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xdd: typ = TYPE_ARRAY self._reserve(4) - n, = struct.unpack_from(">I", self._buffer_view, self._buff_i) + n, = _unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_array_len: raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xde: self._reserve(2) - n, = struct.unpack_from(">H", self._buffer_view, self._buff_i) + n, = _unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP elif b == 0xdf: self._reserve(4) - n, = struct.unpack_from(">I", self._buffer_view, self._buff_i) + n, = _unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_map_len: raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) From 205f7d39b26d7441f3cce86e93449e50bd71a6fa Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 3 Oct 2018 21:06:20 +0900 Subject: [PATCH 0952/1172] Start 0.6 development --- ChangeLog.rst | 6 ++++++ msgpack/_version.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index b6158c3..18a9917 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,9 @@ +0.6.0 +====== + +Release Date: TBD + + 0.5.6 ====== diff --git a/msgpack/_version.py b/msgpack/_version.py index d28f0de..0952ec6 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 5, 6) +version = (0, 6, 0, 'dev') From b077a21f89881df8af56a05cc41d4bdebea19105 Mon Sep 17 00:00:00 2001 From: Marat Sharafutdinov Date: Sun, 4 Nov 2018 19:14:11 +0300 Subject: [PATCH 0953/1172] Fix stream unpacking example in README (#317) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 8925a65..94a4bb2 100644 --- a/README.rst +++ b/README.rst @@ -142,7 +142,7 @@ stream (or from bytes provided through its ``feed`` method). buf = BytesIO() for i in range(100): - buf.write(msgpack.packb(range(i), use_bin_type=True)) + buf.write(msgpack.packb(i, use_bin_type=True)) buf.seek(0) From 91ec9e1daf5cc915a47e2b356a7b1dd9662573a3 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 7 Nov 2018 23:02:35 +0900 Subject: [PATCH 0954/1172] Update travis.yml --- .travis.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 823c8b1..fd6125d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -sudo: false +dist: xenial language: python cache: pip @@ -7,7 +7,8 @@ python: - "3.4" - "3.5" - "3.6" - - "3.7-dev" + - "3.7" + - "nightly" matrix: include: @@ -24,7 +25,12 @@ matrix: - docker pull $DOCKER_IMAGE script: - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh - - python: "pypy" + - python: "pypy2.7-5.10.0" + install: + - pip install -e . + script: + - py.test -v test + - python: "pypy3.5" install: - pip install -e . script: From f6f95972492bcb83d8fe4c63be3b96d46e47bab7 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 3 Oct 2018 20:55:51 +0900 Subject: [PATCH 0955/1172] Merge extension module There were `_packer.so` and `_unpacker.so`. But single module is simpler than double module. Merge extension module into single `_msgpack.so`. --- Makefile | 5 ++--- msgpack/__init__.py | 3 +-- msgpack/_msgpack.pyx | 4 ++++ setup.py | 13 +++---------- 4 files changed, 10 insertions(+), 15 deletions(-) create mode 100644 msgpack/_msgpack.pyx diff --git a/Makefile b/Makefile index 124f243..ff9a482 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: cython .PHONY: cython cython: - cython --cplus msgpack/*.pyx + cython --cplus msgpack/_msgpack.pyx .PHONY: test test: @@ -18,8 +18,7 @@ serve-doc: all .PHONY: clean clean: rm -rf build - rm -f msgpack/_packer.cpp - rm -f msgpack/_unpacker.cpp + rm -f msgpack/_msgpack.cpp rm -rf msgpack/__pycache__ rm -rf test/__pycache__ diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 3955a41..7c5d4c0 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -22,8 +22,7 @@ if os.environ.get('MSGPACK_PUREPYTHON'): from msgpack.fallback import Packer, unpackb, Unpacker else: try: - from msgpack._packer import Packer - from msgpack._unpacker import unpackb, Unpacker + from msgpack._msgpack import Packer, unpackb, Unpacker except ImportError: from msgpack.fallback import Packer, unpackb, Unpacker diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx new file mode 100644 index 0000000..4381394 --- /dev/null +++ b/msgpack/_msgpack.pyx @@ -0,0 +1,4 @@ +# coding: utf-8 +#cython: embedsignature=True, c_string_encoding=ascii +include "_packer.pyx" +include "_unpacker.pyx" diff --git a/setup.py b/setup.py index c252d81..8b8f7bd 100755 --- a/setup.py +++ b/setup.py @@ -68,8 +68,7 @@ if len(version) > 3 and version[3] != 'final': if have_cython: class Sdist(sdist): def __init__(self, *args, **kwargs): - for src in glob('msgpack/*.pyx'): - cythonize(src) + cythonize('msgpack/_msgpack.pyx') sdist.__init__(self, *args, **kwargs) else: Sdist = sdist @@ -85,14 +84,8 @@ else: ext_modules = [] if not hasattr(sys, 'pypy_version_info'): - ext_modules.append(Extension('msgpack._packer', - sources=['msgpack/_packer.cpp'], - libraries=libraries, - include_dirs=['.'], - define_macros=macros, - )) - ext_modules.append(Extension('msgpack._unpacker', - sources=['msgpack/_unpacker.cpp'], + ext_modules.append(Extension('msgpack._msgpack', + sources=['msgpack/_msgpack.cpp'], libraries=libraries, include_dirs=['.'], define_macros=macros, From 6c8e539eec5bfbda752337c34fb701980f061859 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 3 Oct 2018 21:09:45 +0900 Subject: [PATCH 0956/1172] Update travis config --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index fd6125d..822ca9a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: install: - pip install -U pip - pip install cython - - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx + - make cython - docker pull $DOCKER_IMAGE script: - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh @@ -34,19 +34,19 @@ matrix: install: - pip install -e . script: - - py.test -v test + - pytest -v test install: - pip install -U pip - pip install cython - - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx + - make cython - pip install -e . script: - python -c 'import sys; print(hex(sys.maxsize))' - - python -c 'from msgpack import _packer, _unpacker' - - py.test -v test - - MSGPACK_PUREPYTHON=x py.test -v test + - python -c 'from msgpack import _msgpack' + - pytest -v test + - MSGPACK_PUREPYTHON=x pytest -v test # vim: sw=2 ts=2 From 9d11249d89c67fd87acaeb16184414b5ef0b5aa4 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 3 Oct 2018 21:44:06 +0900 Subject: [PATCH 0957/1172] Update docker/runtests --- docker/runtests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/runtests.sh b/docker/runtests.sh index 113b630..f6c2c68 100755 --- a/docker/runtests.sh +++ b/docker/runtests.sh @@ -8,7 +8,7 @@ for V in cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do $PYBIN/pip install pytest pushd test # prevent importing msgpack package in current directory. $PYBIN/python -c 'import sys; print(hex(sys.maxsize))' - $PYBIN/python -c 'from msgpack import _packer, _unpacker' + $PYBIN/python -c 'from msgpack import _msgpack' # Ensure extension is available $PYBIN/pytest -v . popd done From ae90b26c3015e090a80a880b73895daa35f048fa Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 8 Nov 2018 22:21:05 +0900 Subject: [PATCH 0958/1172] Update ChangeLog --- ChangeLog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 18a9917..b135745 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -3,6 +3,10 @@ Release Date: TBD +Extension modules are merged. There is ``msgpack._msgpack`` instead of +``msgpack._packer`` and ``msgpack._unpacker``. (#314) + + 0.5.6 ====== From 3b80233592674d18c8db7a62fa56504a5a285296 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 8 Nov 2018 22:21:44 +0900 Subject: [PATCH 0959/1172] unpacker: Make default size limit smaller (#319) To avoid DoS attack, make default size limit smaller. Fixes #295 --- msgpack/_unpacker.pyx | 31 +++++++++++++++++-------------- msgpack/fallback.py | 20 +++++++++++--------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index d7fa5bc..cc9e7f0 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -162,11 +162,11 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=True, bint raw=True, encoding=None, unicode_errors=None, object_pairs_hook=None, ext_hook=ExtType, - Py_ssize_t max_str_len=2147483647, # 2**32-1 - Py_ssize_t max_bin_len=2147483647, - Py_ssize_t max_array_len=2147483647, - Py_ssize_t max_map_len=2147483647, - Py_ssize_t max_ext_len=2147483647): + Py_ssize_t max_str_len=1024*1024, + Py_ssize_t max_bin_len=1024*1024, + Py_ssize_t max_array_len=128*1024, + Py_ssize_t max_map_len=32*1024, + Py_ssize_t max_ext_len=1024*1024): """ Unpack packed_bytes to object. Returns an unpacked object. @@ -261,16 +261,19 @@ cdef class Unpacker(object): You should set this parameter when unpacking data from untrusted source. :param int max_str_len: - Limits max length of str. (default: 2**31-1) + Limits max length of str. (default: 1024*1024) :param int max_bin_len: - Limits max length of bin. (default: 2**31-1) + Limits max length of bin. (default: 1024*1024) :param int max_array_len: - Limits max length of array. (default: 2**31-1) + Limits max length of array. (default: 128*1024) :param int max_map_len: - Limits max length of map. (default: 2**31-1) + Limits max length of map. (default: 32*1024) + + :param int max_ext_len: + Limits max length of map. (default: 1024*1024) :param str encoding: Deprecated, use raw instead. @@ -322,11 +325,11 @@ cdef class Unpacker(object): object object_hook=None, object object_pairs_hook=None, object list_hook=None, encoding=None, unicode_errors=None, int max_buffer_size=0, object ext_hook=ExtType, - Py_ssize_t max_str_len=2147483647, # 2**32-1 - Py_ssize_t max_bin_len=2147483647, - Py_ssize_t max_array_len=2147483647, - Py_ssize_t max_map_len=2147483647, - Py_ssize_t max_ext_len=2147483647): + Py_ssize_t max_str_len=1024*1024, + Py_ssize_t max_bin_len=1024*1024, + Py_ssize_t max_array_len=128*1024, + Py_ssize_t max_map_len=32*1024, + Py_ssize_t max_ext_len=1024*1024): cdef const char *cenc=NULL, cdef const char *cerr=NULL diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 0b60ba2..895864e 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -184,17 +184,19 @@ class Unpacker(object): You should set this parameter when unpacking data from untrusted source. :param int max_str_len: - Limits max length of str. (default: 2**31-1) + Limits max length of str. (default: 1024*1024) :param int max_bin_len: - Limits max length of bin. (default: 2**31-1) + Limits max length of bin. (default: 1024*1024) :param int max_array_len: - Limits max length of array. (default: 2**31-1) + Limits max length of array. (default: 128*1024) :param int max_map_len: - Limits max length of map. (default: 2**31-1) + Limits max length of map. (default: 32*1024) + :param int max_ext_len: + Limits max length of map. (default: 1024*1024) example of streaming deserialize from file-like object:: @@ -218,11 +220,11 @@ class Unpacker(object): object_hook=None, object_pairs_hook=None, list_hook=None, encoding=None, unicode_errors=None, max_buffer_size=0, ext_hook=ExtType, - max_str_len=2147483647, # 2**32-1 - max_bin_len=2147483647, - max_array_len=2147483647, - max_map_len=2147483647, - max_ext_len=2147483647): + max_str_len=1024*1024, + max_bin_len=1024*1024, + max_array_len=128*1024, + max_map_len=32*1024, + max_ext_len=1024*1024): if encoding is not None: warnings.warn( From a8b3e97fe588a2411a8e869b52be1946ed9f0f86 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 8 Nov 2018 22:25:05 +0900 Subject: [PATCH 0960/1172] Update changelog --- ChangeLog.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index b135745..d71e0f5 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -3,9 +3,21 @@ Release Date: TBD + +Important changes +------------------ + Extension modules are merged. There is ``msgpack._msgpack`` instead of ``msgpack._packer`` and ``msgpack._unpacker``. (#314) +unpacker: Default size limits is smaller than before to avoid DoS attack. +If you need to handle large data, you need to specify limits manually. + + + +Other changes +-------------- + 0.5.6 From 9e210bfc1a922031db67bf42e508b1b4550814c6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 9 Nov 2018 20:55:13 +0900 Subject: [PATCH 0961/1172] Add Packer.buffer() (#320) --- ChangeLog.rst | 2 ++ docs/Makefile | 2 +- docs/advanced.rst | 32 ++++++++++++++++++++++++++++++++ docs/index.rst | 1 + msgpack/_packer.pyx | 14 ++++++++++++-- msgpack/buff_converter.h | 28 ++++++++++++++++++++++++++++ msgpack/fallback.py | 36 ++++++++++++++++++++---------------- test/test_pack.py | 12 +++++++++++- 8 files changed, 107 insertions(+), 20 deletions(-) create mode 100644 docs/advanced.rst create mode 100644 msgpack/buff_converter.h diff --git a/ChangeLog.rst b/ChangeLog.rst index d71e0f5..cc6b5e4 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -18,6 +18,8 @@ If you need to handle large data, you need to specify limits manually. Other changes -------------- +Add ``Unpacker.getbuffer()`` method. + 0.5.6 diff --git a/docs/Makefile b/docs/Makefile index b09d884..831a6a7 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -153,7 +153,7 @@ doctest: "results in $(BUILDDIR)/doctest/output.txt." serve: html - cd _build/html && python3 -m http.server + python3 -m http.server -d _build/html zip: html cd _build/html && zip -r ../../../msgpack-doc.zip . diff --git a/docs/advanced.rst b/docs/advanced.rst new file mode 100644 index 0000000..3837008 --- /dev/null +++ b/docs/advanced.rst @@ -0,0 +1,32 @@ +Advanced usage +=============== + +Packer +------ + +autoreset +~~~~~~~~~ + +When you used ``autoreset=False`` option of :class:`~msgpack.Packer`, +``pack()`` method doesn't return packed ``bytes``. + +You can use :meth:`~msgpack.Packer.bytes` or :meth:`~msgpack.Packer.getbuffer` to +get packed data. + +``bytes()`` returns ``bytes`` object. ``getbuffer()`` returns some bytes-like +object. It's concrete type is implement detail and it will be changed in future +versions. + +You can reduce temporary bytes object by using ``Unpacker.getbuffer()``. + +.. code-block:: python + + packer = Packer(use_bin_type=True, autoreset=False) + + packer.pack([1, 2]) + packer.pack([3, 4]) + + with open('data.bin', 'wb') as f: + f.write(packer.getbuffer()) + + packer.reset() # reset internal buffer diff --git a/docs/index.rst b/docs/index.rst index dcdab4f..e9c2ce8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,3 +8,4 @@ language data exchange. :maxdepth: 1 api + advanced diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 225f24a..fd05ae0 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -41,6 +41,9 @@ cdef extern from "pack.h": int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l) int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit) +cdef extern from "buff_converter.h": + object buff_to_buff(char *, Py_ssize_t) + cdef int DEFAULT_RECURSE_LIMIT=511 cdef long long ITEM_LIMIT = (2**32)-1 @@ -349,9 +352,16 @@ cdef class Packer(object): return buf def reset(self): - """Clear internal buffer.""" + """Reset internal buffer. + + This method is usaful only when autoreset=False. + """ self.pk.length = 0 def bytes(self): - """Return buffer content.""" + """Return internal buffer contents as bytes object""" return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) + + def getbuffer(self): + """Return view of internal buffer.""" + return buff_to_buff(self.pk.buf, self.pk.length) diff --git a/msgpack/buff_converter.h b/msgpack/buff_converter.h new file mode 100644 index 0000000..bc7227a --- /dev/null +++ b/msgpack/buff_converter.h @@ -0,0 +1,28 @@ +#include "Python.h" + +/* cython does not support this preprocessor check => write it in raw C */ +#if PY_MAJOR_VERSION == 2 +static PyObject * +buff_to_buff(char *buff, Py_ssize_t size) +{ + return PyBuffer_FromMemory(buff, size); +} + +#elif (PY_MAJOR_VERSION == 3) && (PY_MINOR_VERSION >= 3) +static PyObject * +buff_to_buff(char *buff, Py_ssize_t size) +{ + return PyMemoryView_FromMemory(buff, size, PyBUF_READ); +} +#else +static PyObject * +buff_to_buff(char *buff, Py_ssize_t size) +{ + Py_buffer pybuf; + if (PyBuffer_FillInfo(&pybuf, NULL, buff, size, 1, PyBUF_FULL_RO) == -1) { + return NULL; + } + + return PyMemoryView_FromBuffer(&pybuf); +} +#endif diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 895864e..5b4d6ce 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -860,43 +860,35 @@ class Packer(object): except: self._buffer = StringIO() # force reset raise - ret = self._buffer.getvalue() if self._autoreset: + ret = self._buffer.getvalue() self._buffer = StringIO() - elif USING_STRINGBUILDER: - self._buffer = StringIO(ret) - return ret + return ret def pack_map_pairs(self, pairs): self._pack_map_pairs(len(pairs), pairs) - ret = self._buffer.getvalue() if self._autoreset: + ret = self._buffer.getvalue() self._buffer = StringIO() - elif USING_STRINGBUILDER: - self._buffer = StringIO(ret) - return ret + return ret def pack_array_header(self, n): if n >= 2**32: raise PackValueError self._pack_array_header(n) - ret = self._buffer.getvalue() if self._autoreset: + ret = self._buffer.getvalue() self._buffer = StringIO() - elif USING_STRINGBUILDER: - self._buffer = StringIO(ret) - return ret + return ret def pack_map_header(self, n): if n >= 2**32: raise PackValueError self._pack_map_header(n) - ret = self._buffer.getvalue() if self._autoreset: + ret = self._buffer.getvalue() self._buffer = StringIO() - elif USING_STRINGBUILDER: - self._buffer = StringIO(ret) - return ret + return ret def pack_ext_type(self, typecode, data): if not isinstance(typecode, int): @@ -976,7 +968,19 @@ class Packer(object): raise PackValueError('Bin is too large') def bytes(self): + """Return internal buffer contents as bytes object""" return self._buffer.getvalue() def reset(self): + """Reset internal buffer. + + This method is usaful only when autoreset=False. + """ self._buffer = StringIO() + + def getbuffer(self): + """Return view of internal buffer.""" + if USING_STRINGBUILDER or not PY3: + return memoryview(self.bytes()) + else: + return self._buffer.getbuffer() diff --git a/test/test_pack.py b/test/test_pack.py index b447f9c..4608083 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -5,7 +5,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera import struct from pytest import raises, xfail -from msgpack import packb, unpackb, Unpacker, Packer +from msgpack import packb, unpackb, Unpacker, Packer, pack from collections import OrderedDict from io import BytesIO @@ -148,3 +148,13 @@ def test_pairlist(): packed = packer.pack_map_pairs(pairlist) unpacked = unpackb(packed, object_pairs_hook=list) assert pairlist == unpacked + +def test_get_buffer(): + packer = Packer(autoreset=0, use_bin_type=True) + packer.pack([1, 2]) + strm = BytesIO() + strm.write(packer.getbuffer()) + written = strm.getvalue() + + expected = packb([1, 2], use_bin_type=True) + assert written == expected From 1bf62ba6f8f94ab8a7dd135e0039ee3b10e0e96c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Fri, 9 Nov 2018 21:39:25 +0900 Subject: [PATCH 0962/1172] PendingDeprecationWarning -> DeprecationWarning (#321) --- msgpack/_packer.pyx | 2 +- msgpack/_unpacker.pyx | 6 +++--- msgpack/fallback.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index fd05ae0..6a6d917 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -121,7 +121,7 @@ cdef class Packer(object): bint use_single_float=False, bint autoreset=True, bint use_bin_type=False, bint strict_types=False): if encoding is not None: - PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated.", 1) + PyErr_WarnEx(DeprecationWarning, "encoding is deprecated.", 1) self.use_float = use_single_float self.strict_types = strict_types self.autoreset = autoreset diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index cc9e7f0..85c404a 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -186,7 +186,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef int new_protocol = 0 if encoding is not None: - PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) + PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) cenc = encoding if unicode_errors is not None: @@ -213,7 +213,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, def unpack(object stream, **kwargs): PyErr_WarnEx( - PendingDeprecationWarning, + DeprecationWarning, "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", 1) data = stream.read() return unpackb(data, **kwargs) @@ -360,7 +360,7 @@ cdef class Unpacker(object): self.stream_offset = 0 if encoding is not None: - PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) + PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) self.encoding = encoding cenc = encoding diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 5b4d6ce..197b6d2 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -103,7 +103,7 @@ def _get_data_from_buffer(obj): def unpack(stream, **kwargs): warnings.warn( "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", - PendingDeprecationWarning) + DeprecationWarning) data = stream.read() return unpackb(data, **kwargs) @@ -229,7 +229,7 @@ class Unpacker(object): if encoding is not None: warnings.warn( "encoding is deprecated, Use raw=False instead.", - PendingDeprecationWarning) + DeprecationWarning) if unicode_errors is None: unicode_errors = 'strict' @@ -727,7 +727,7 @@ class Packer(object): else: warnings.warn( "encoding is deprecated, Use raw=False instead.", - PendingDeprecationWarning) + DeprecationWarning) if unicode_errors is None: unicode_errors = 'strict' From 07f0beeabb71828377f481ff83746997b3babf23 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 12 Nov 2018 02:19:01 +0900 Subject: [PATCH 0963/1172] Remove deprecated exception classes (#323) --- msgpack/_packer.pyx | 27 ++++++++------- msgpack/_unpacker.pyx | 40 ++++++++++------------ msgpack/exceptions.py | 35 +++++++++----------- msgpack/fallback.py | 77 +++++++++++++++++++++---------------------- 4 files changed, 84 insertions(+), 95 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 6a6d917..2643f85 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -5,7 +5,6 @@ from cpython cimport * from cpython.version cimport PY_MAJOR_VERSION from cpython.exc cimport PyErr_WarnEx -from msgpack.exceptions import PackValueError, PackOverflowError from msgpack import ExtType @@ -165,7 +164,7 @@ cdef class Packer(object): cdef Py_buffer view if nest_limit < 0: - raise PackValueError("recursion limit exceeded.") + raise ValueError("recursion limit exceeded.") while True: if o is None: @@ -191,7 +190,7 @@ cdef class Packer(object): default_used = True continue else: - raise PackOverflowError("Integer value out of range") + raise OverflowError("Integer value out of range") elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o): longval = o ret = msgpack_pack_long(&self.pk, longval) @@ -205,7 +204,7 @@ cdef class Packer(object): elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): L = len(o) if L > ITEM_LIMIT: - raise PackValueError("%s is too large" % type(o).__name__) + raise ValueError("%s is too large" % type(o).__name__) rawval = o ret = msgpack_pack_bin(&self.pk, L) if ret == 0: @@ -214,12 +213,12 @@ cdef class Packer(object): if self.encoding == NULL and self.unicode_errors == NULL: ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); if ret == -2: - raise PackValueError("unicode string is too large") + raise ValueError("unicode string is too large") else: o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) L = len(o) if L > ITEM_LIMIT: - raise PackValueError("unicode string is too large") + raise ValueError("unicode string is too large") ret = msgpack_pack_raw(&self.pk, L) if ret == 0: rawval = o @@ -228,7 +227,7 @@ cdef class Packer(object): d = o L = len(d) if L > ITEM_LIMIT: - raise PackValueError("dict is too large") + raise ValueError("dict is too large") ret = msgpack_pack_map(&self.pk, L) if ret == 0: for k, v in d.iteritems(): @@ -239,7 +238,7 @@ cdef class Packer(object): elif not strict_types and PyDict_Check(o): L = len(o) if L > ITEM_LIMIT: - raise PackValueError("dict is too large") + raise ValueError("dict is too large") ret = msgpack_pack_map(&self.pk, L) if ret == 0: for k, v in o.items(): @@ -253,13 +252,13 @@ cdef class Packer(object): rawval = o.data L = len(o.data) if L > ITEM_LIMIT: - raise PackValueError("EXT data is too large") + raise ValueError("EXT data is too large") ret = msgpack_pack_ext(&self.pk, longval, L) ret = msgpack_pack_raw_body(&self.pk, rawval, L) elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): L = len(o) if L > ITEM_LIMIT: - raise PackValueError("list is too large") + raise ValueError("list is too large") ret = msgpack_pack_array(&self.pk, L) if ret == 0: for v in o: @@ -267,11 +266,11 @@ cdef class Packer(object): if ret != 0: break elif PyMemoryView_Check(o): if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: - raise PackValueError("could not get buffer for memoryview") + raise ValueError("could not get buffer for memoryview") L = view.len if L > ITEM_LIMIT: PyBuffer_Release(&view); - raise PackValueError("memoryview is too large") + raise ValueError("memoryview is too large") ret = msgpack_pack_bin(&self.pk, L) if ret == 0: ret = msgpack_pack_raw_body(&self.pk, view.buf, L) @@ -304,7 +303,7 @@ cdef class Packer(object): def pack_array_header(self, long long size): if size > ITEM_LIMIT: - raise PackValueError + raise ValueError cdef int ret = msgpack_pack_array(&self.pk, size) if ret == -1: raise MemoryError @@ -317,7 +316,7 @@ cdef class Packer(object): def pack_map_header(self, long long size): if size > ITEM_LIMIT: - raise PackValueError + raise ValueError cdef int ret = msgpack_pack_map(&self.pk, size) if ret == -1: raise MemoryError diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 85c404a..2f99019 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -35,7 +35,6 @@ ctypedef unsigned long long uint64_t from msgpack.exceptions import ( BufferFull, OutOfData, - UnpackValueError, ExtraData, ) from msgpack import ExtType @@ -208,7 +207,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) return obj unpack_clear(&ctx) - raise UnpackValueError("Unpack failed: error = %d" % (ret,)) + raise ValueError("Unpack failed: error = %d" % (ret,)) def unpack(object stream, **kwargs): @@ -460,28 +459,25 @@ cdef class Unpacker(object): else: raise OutOfData("No more data to unpack.") - try: - ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) - self.stream_offset += self.buf_head - prev_head - if write_bytes is not None: - write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head)) + ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + self.stream_offset += self.buf_head - prev_head + if write_bytes is not None: + write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head)) - if ret == 1: - obj = unpack_data(&self.ctx) - unpack_init(&self.ctx) - return obj - elif ret == 0: - if self.file_like is not None: - self.read_from_file() - continue - if iter: - raise StopIteration("No more data to unpack.") - else: - raise OutOfData("No more data to unpack.") + if ret == 1: + obj = unpack_data(&self.ctx) + unpack_init(&self.ctx) + return obj + elif ret == 0: + if self.file_like is not None: + self.read_from_file() + continue + if iter: + raise StopIteration("No more data to unpack.") else: - raise UnpackValueError("Unpack failed: error = %d" % (ret,)) - except ValueError as e: - raise UnpackValueError(e) + raise OutOfData("No more data to unpack.") + else: + raise ValueError("Unpack failed: error = %d" % (ret,)) def read_bytes(self, Py_ssize_t nbytes): """Read a specified number of raw bytes from the stream""" diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py index 9766881..5bee5b2 100644 --- a/msgpack/exceptions.py +++ b/msgpack/exceptions.py @@ -1,6 +1,10 @@ class UnpackException(Exception): - """Deprecated. Use Exception instead to catch all exception during unpacking.""" + """Base class for some exceptions raised while unpacking. + NOTE: unpack may raise exception other than subclass of + UnpackException. If you want to catch all error, catch + Exception instead. + """ class BufferFull(UnpackException): pass @@ -10,11 +14,16 @@ class OutOfData(UnpackException): pass -class UnpackValueError(UnpackException, ValueError): - """Deprecated. Use ValueError instead.""" +# Deprecated. Use ValueError instead +UnpackValueError = ValueError class ExtraData(UnpackValueError): + """ExtraData is raised when there is trailing data. + + This exception is raised while only one-shot (not streaming) + unpack. + """ def __init__(self, unpacked, extra): self.unpacked = unpacked self.extra = extra @@ -23,19 +32,7 @@ class ExtraData(UnpackValueError): return "unpack(b) received extra data." -class PackException(Exception): - """Deprecated. Use Exception instead to catch all exception during packing.""" - - -class PackValueError(PackException, ValueError): - """PackValueError is raised when type of input data is supported but it's value is unsupported. - - Deprecated. Use ValueError instead. - """ - - -class PackOverflowError(PackValueError, OverflowError): - """PackOverflowError is raised when integer value is out of range of msgpack support [-2**31, 2**32). - - Deprecated. Use ValueError instead. - """ +#Deprecated. Use Exception instead to catch all exception during packing. +PackException = Exception +PackValueError = ValueError +PackOverflowError = OverflowError diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 197b6d2..9d46171 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -52,9 +52,6 @@ else: from msgpack.exceptions import ( BufferFull, OutOfData, - UnpackValueError, - PackValueError, - PackOverflowError, ExtraData) from msgpack import ExtType @@ -120,7 +117,7 @@ def unpackb(packed, **kwargs): try: ret = unpacker._unpack() except OutOfData: - raise UnpackValueError("Data is not enough.") + raise ValueError("Data is not enough.") if unpacker._got_extradata(): raise ExtraData(ret, unpacker._get_extradata()) return ret @@ -370,18 +367,18 @@ class Unpacker(object): n = b & 0b00011111 typ = TYPE_RAW if n > self._max_str_len: - raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) elif b & 0b11110000 == 0b10010000: n = b & 0b00001111 typ = TYPE_ARRAY if n > self._max_array_len: - raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) + raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b & 0b11110000 == 0b10000000: n = b & 0b00001111 typ = TYPE_MAP if n > self._max_map_len: - raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) + raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) elif b == 0xc0: obj = None elif b == 0xc2: @@ -394,7 +391,7 @@ class Unpacker(object): n = self._buffer[self._buff_i] self._buff_i += 1 if n > self._max_bin_len: - raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) + raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._read(n) elif b == 0xc5: typ = TYPE_BIN @@ -402,7 +399,7 @@ class Unpacker(object): n = _unpack_from(">H", self._buffer, self._buff_i)[0] self._buff_i += 2 if n > self._max_bin_len: - raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) + raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._read(n) elif b == 0xc6: typ = TYPE_BIN @@ -410,7 +407,7 @@ class Unpacker(object): n = _unpack_from(">I", self._buffer, self._buff_i)[0] self._buff_i += 4 if n > self._max_bin_len: - raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) + raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._read(n) elif b == 0xc7: # ext 8 typ = TYPE_EXT @@ -418,7 +415,7 @@ class Unpacker(object): L, n = _unpack_from('Bb', self._buffer, self._buff_i) self._buff_i += 2 if L > self._max_ext_len: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) elif b == 0xc8: # ext 16 typ = TYPE_EXT @@ -426,7 +423,7 @@ class Unpacker(object): L, n = _unpack_from('>Hb', self._buffer, self._buff_i) self._buff_i += 3 if L > self._max_ext_len: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) elif b == 0xc9: # ext 32 typ = TYPE_EXT @@ -434,7 +431,7 @@ class Unpacker(object): L, n = _unpack_from('>Ib', self._buffer, self._buff_i) self._buff_i += 5 if L > self._max_ext_len: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) elif b == 0xca: self._reserve(4) @@ -479,35 +476,35 @@ class Unpacker(object): elif b == 0xd4: # fixext 1 typ = TYPE_EXT if self._max_ext_len < 1: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) self._reserve(2) n, obj = _unpack_from("b1s", self._buffer, self._buff_i) self._buff_i += 2 elif b == 0xd5: # fixext 2 typ = TYPE_EXT if self._max_ext_len < 2: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) self._reserve(3) n, obj = _unpack_from("b2s", self._buffer, self._buff_i) self._buff_i += 3 elif b == 0xd6: # fixext 4 typ = TYPE_EXT if self._max_ext_len < 4: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) self._reserve(5) n, obj = _unpack_from("b4s", self._buffer, self._buff_i) self._buff_i += 5 elif b == 0xd7: # fixext 8 typ = TYPE_EXT if self._max_ext_len < 8: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) self._reserve(9) n, obj = _unpack_from("b8s", self._buffer, self._buff_i) self._buff_i += 9 elif b == 0xd8: # fixext 16 typ = TYPE_EXT if self._max_ext_len < 16: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) self._reserve(17) n, obj = _unpack_from("b16s", self._buffer, self._buff_i) self._buff_i += 17 @@ -517,7 +514,7 @@ class Unpacker(object): n = self._buffer[self._buff_i] self._buff_i += 1 if n > self._max_str_len: - raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) elif b == 0xda: typ = TYPE_RAW @@ -525,7 +522,7 @@ class Unpacker(object): n, = _unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_str_len: - raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) elif b == 0xdb: typ = TYPE_RAW @@ -533,7 +530,7 @@ class Unpacker(object): n, = _unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_str_len: - raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) elif b == 0xdc: typ = TYPE_ARRAY @@ -541,30 +538,30 @@ class Unpacker(object): n, = _unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_array_len: - raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) + raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xdd: typ = TYPE_ARRAY self._reserve(4) n, = _unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_array_len: - raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) + raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xde: self._reserve(2) n, = _unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_map_len: - raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) + raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP elif b == 0xdf: self._reserve(4) n, = _unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_map_len: - raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) + raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP else: - raise UnpackValueError("Unknown header: 0x%x" % b) + raise ValueError("Unknown header: 0x%x" % b) return typ, n, obj def _unpack(self, execute=EX_CONSTRUCT): @@ -572,11 +569,11 @@ class Unpacker(object): if execute == EX_READ_ARRAY_HEADER: if typ != TYPE_ARRAY: - raise UnpackValueError("Expected array") + raise ValueError("Expected array") return n if execute == EX_READ_MAP_HEADER: if typ != TYPE_MAP: - raise UnpackValueError("Expected map") + raise ValueError("Expected map") return n # TODO should we eliminate the recursion? if typ == TYPE_ARRAY: @@ -754,7 +751,7 @@ class Packer(object): list_types = (list, tuple) while True: if nest_limit < 0: - raise PackValueError("recursion limit exceeded") + raise ValueError("recursion limit exceeded") if obj is None: return self._buffer.write(b"\xc0") if check(obj, bool): @@ -786,11 +783,11 @@ class Packer(object): obj = self._default(obj) default_used = True continue - raise PackOverflowError("Integer value out of range") + raise OverflowError("Integer value out of range") if check(obj, (bytes, bytearray)): n = len(obj) if n >= 2**32: - raise PackValueError("%s is too large" % type(obj).__name__) + raise ValueError("%s is too large" % type(obj).__name__) self._pack_bin_header(n) return self._buffer.write(obj) if check(obj, Unicode): @@ -801,13 +798,13 @@ class Packer(object): obj = obj.encode(self._encoding, self._unicode_errors) n = len(obj) if n >= 2**32: - raise PackValueError("String is too large") + raise ValueError("String is too large") self._pack_raw_header(n) return self._buffer.write(obj) if check(obj, memoryview): n = len(obj) * obj.itemsize if n >= 2**32: - raise PackValueError("Memoryview is too large") + raise ValueError("Memoryview is too large") self._pack_bin_header(n) return self._buffer.write(obj) if check(obj, float): @@ -874,7 +871,7 @@ class Packer(object): def pack_array_header(self, n): if n >= 2**32: - raise PackValueError + raise ValueError self._pack_array_header(n) if self._autoreset: ret = self._buffer.getvalue() @@ -883,7 +880,7 @@ class Packer(object): def pack_map_header(self, n): if n >= 2**32: - raise PackValueError + raise ValueError self._pack_map_header(n) if self._autoreset: ret = self._buffer.getvalue() @@ -899,7 +896,7 @@ class Packer(object): raise TypeError("data must have bytes type") L = len(data) if L > 0xffffffff: - raise PackValueError("Too large data") + raise ValueError("Too large data") if L == 1: self._buffer.write(b'\xd4') elif L == 2: @@ -926,7 +923,7 @@ class Packer(object): return self._buffer.write(struct.pack(">BH", 0xdc, n)) if n <= 0xffffffff: return self._buffer.write(struct.pack(">BI", 0xdd, n)) - raise PackValueError("Array is too large") + raise ValueError("Array is too large") def _pack_map_header(self, n): if n <= 0x0f: @@ -935,7 +932,7 @@ class Packer(object): return self._buffer.write(struct.pack(">BH", 0xde, n)) if n <= 0xffffffff: return self._buffer.write(struct.pack(">BI", 0xdf, n)) - raise PackValueError("Dict is too large") + raise ValueError("Dict is too large") def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT): self._pack_map_header(n) @@ -953,7 +950,7 @@ class Packer(object): elif n <= 0xffffffff: self._buffer.write(struct.pack(">BI", 0xdb, n)) else: - raise PackValueError('Raw is too large') + raise ValueError('Raw is too large') def _pack_bin_header(self, n): if not self._use_bin_type: @@ -965,7 +962,7 @@ class Packer(object): elif n <= 0xffffffff: return self._buffer.write(struct.pack(">BI", 0xc6, n)) else: - raise PackValueError('Bin is too large') + raise ValueError('Bin is too large') def bytes(self): """Return internal buffer contents as bytes object""" From 39f8aa78c7bc6b6b18a1c814a0c296f55242f028 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 12 Nov 2018 02:33:31 +0900 Subject: [PATCH 0964/1172] Remove deprecated write_bytes option (#322) --- msgpack/_unpacker.pyx | 35 ++++++++++++----------------------- msgpack/fallback.py | 20 ++++---------------- test/test_unpack_raw.py | 29 ----------------------------- 3 files changed, 16 insertions(+), 68 deletions(-) delete mode 100644 test/test_unpack_raw.py diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 2f99019..e168587 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -135,10 +135,10 @@ cdef inline int get_data_from_buffer(object obj, if view.itemsize != 1: PyBuffer_Release(view) raise BufferError("cannot unpack from multi-byte object") - if PyBuffer_IsContiguous(view, 'A') == 0: + if PyBuffer_IsContiguous(view, b'A') == 0: PyBuffer_Release(view) # create a contiguous copy and get buffer - contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, 'C') + contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, b'C') PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE) # view must hold the only reference to contiguous, # so memory is freed when view is released @@ -440,14 +440,11 @@ cdef class Unpacker(object): else: self.file_like = None - cdef object _unpack(self, execute_fn execute, object write_bytes, bint iter=0): + cdef object _unpack(self, execute_fn execute, bint iter=0): cdef int ret cdef object obj cdef Py_ssize_t prev_head - if write_bytes is not None: - PyErr_WarnEx(DeprecationWarning, "`write_bytes` option is deprecated. Use `.tell()` instead.", 1) - if self.buf_head >= self.buf_tail and self.file_like is not None: self.read_from_file() @@ -461,8 +458,6 @@ cdef class Unpacker(object): ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) self.stream_offset += self.buf_head - prev_head - if write_bytes is not None: - write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head)) if ret == 1: obj = unpack_data(&self.ctx) @@ -489,41 +484,35 @@ cdef class Unpacker(object): ret += self.file_like.read(nbytes - len(ret)) return ret - def unpack(self, object write_bytes=None): + def unpack(self): """Unpack one object - If write_bytes is not None, it will be called with parts of the raw - message as it is unpacked. - Raises `OutOfData` when there are no more bytes to unpack. """ - return self._unpack(unpack_construct, write_bytes) + return self._unpack(unpack_construct) - def skip(self, object write_bytes=None): + def skip(self): """Read and ignore one object, returning None - If write_bytes is not None, it will be called with parts of the raw - message as it is unpacked. - Raises `OutOfData` when there are no more bytes to unpack. """ - return self._unpack(unpack_skip, write_bytes) + return self._unpack(unpack_skip) - def read_array_header(self, object write_bytes=None): + def read_array_header(self): """assuming the next object is an array, return its size n, such that the next n unpack() calls will iterate over its contents. Raises `OutOfData` when there are no more bytes to unpack. """ - return self._unpack(read_array_header, write_bytes) + return self._unpack(read_array_header) - def read_map_header(self, object write_bytes=None): + def read_map_header(self): """assuming the next object is a map, return its size n, such that the next n * 2 unpack() calls will iterate over its key-value pairs. Raises `OutOfData` when there are no more bytes to unpack. """ - return self._unpack(read_map_header, write_bytes) + return self._unpack(read_map_header) def tell(self): return self.stream_offset @@ -532,7 +521,7 @@ cdef class Unpacker(object): return self def __next__(self): - return self._unpack(unpack_construct, None, 1) + return self._unpack(unpack_construct, 1) # for debug. #def _buf(self): diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 9d46171..b9ef296 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -640,34 +640,22 @@ class Unpacker(object): next = __next__ - def skip(self, write_bytes=None): + def skip(self): self._unpack(EX_SKIP) - if write_bytes is not None: - warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning) - write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) self._consume() - def unpack(self, write_bytes=None): + def unpack(self): ret = self._unpack(EX_CONSTRUCT) - if write_bytes is not None: - warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning) - write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) self._consume() return ret - def read_array_header(self, write_bytes=None): + def read_array_header(self): ret = self._unpack(EX_READ_ARRAY_HEADER) - if write_bytes is not None: - warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning) - write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) self._consume() return ret - def read_map_header(self, write_bytes=None): + def read_map_header(self): ret = self._unpack(EX_READ_MAP_HEADER) - if write_bytes is not None: - warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning) - write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) self._consume() return ret diff --git a/test/test_unpack_raw.py b/test/test_unpack_raw.py deleted file mode 100644 index 7002601..0000000 --- a/test/test_unpack_raw.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Tests for cases where the user seeks to obtain packed msgpack objects""" - -import io -from msgpack import Unpacker, packb - - -def test_write_bytes(): - unpacker = Unpacker() - unpacker.feed(b'abc') - f = io.BytesIO() - assert unpacker.unpack(f.write) == ord('a') - assert f.getvalue() == b'a' - f = io.BytesIO() - assert unpacker.skip(f.write) is None - assert f.getvalue() == b'b' - f = io.BytesIO() - assert unpacker.skip() is None - assert f.getvalue() == b'' - - -def test_write_bytes_multi_buffer(): - long_val = (5) * 100 - expected = packb(long_val) - unpacker = Unpacker(io.BytesIO(expected), read_size=3, max_buffer_size=3) - - f = io.BytesIO() - unpacked = unpacker.unpack(f.write) - assert unpacked == long_val - assert f.getvalue() == expected From 2b5f59166beeccde0ee230c8673cf50932c8daba Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 14 Nov 2018 16:34:51 +0900 Subject: [PATCH 0965/1172] fallback: Fix warning stacklevel (#327) --- msgpack/fallback.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index b9ef296..04fb5b9 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -89,7 +89,7 @@ def _get_data_from_buffer(obj): warnings.warn("using old buffer interface to unpack %s; " "this leads to unpacking errors if slicing is used and " "will be removed in a future version" % type(obj), - RuntimeWarning) + RuntimeWarning, stacklevel=3) else: raise if view.itemsize != 1: @@ -100,7 +100,7 @@ def _get_data_from_buffer(obj): def unpack(stream, **kwargs): warnings.warn( "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", - DeprecationWarning) + DeprecationWarning, stacklevel=2) data = stream.read() return unpackb(data, **kwargs) @@ -226,7 +226,7 @@ class Unpacker(object): if encoding is not None: warnings.warn( "encoding is deprecated, Use raw=False instead.", - DeprecationWarning) + DeprecationWarning, stacklevel=2) if unicode_errors is None: unicode_errors = 'strict' @@ -712,7 +712,7 @@ class Packer(object): else: warnings.warn( "encoding is deprecated, Use raw=False instead.", - DeprecationWarning) + DeprecationWarning, stacklevel=2) if unicode_errors is None: unicode_errors = 'strict' From d782464c9150e448ab3a8d81197ff335e1ac2c2b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 14 Nov 2018 16:35:37 +0900 Subject: [PATCH 0966/1172] Refactor Cython code (#328) _msgpack -> _cmsgpack --- .travis.yml | 2 +- Makefile | 2 +- appveyor.yml | 3 ++- docker/runtests.sh | 2 +- msgpack/__init__.py | 2 +- msgpack/_cmsgpack.pyx | 4 ++++ msgpack/_msgpack.pyx | 4 ---- msgpack/_packer.pyx | 10 +++------- msgpack/_unpacker.pyx | 22 +--------------------- setup.py | 6 +++--- 10 files changed, 17 insertions(+), 40 deletions(-) create mode 100644 msgpack/_cmsgpack.pyx delete mode 100644 msgpack/_msgpack.pyx diff --git a/.travis.yml b/.travis.yml index 822ca9a..1adbdc2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,7 +45,7 @@ install: script: - python -c 'import sys; print(hex(sys.maxsize))' - - python -c 'from msgpack import _msgpack' + - python -c 'from msgpack import _cmsgpack' - pytest -v test - MSGPACK_PUREPYTHON=x pytest -v test diff --git a/Makefile b/Makefile index ff9a482..b65aa85 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: cython .PHONY: cython cython: - cython --cplus msgpack/_msgpack.pyx + cython --cplus msgpack/_cmsgpack.pyx .PHONY: test test: diff --git a/appveyor.yml b/appveyor.yml index 72b334a..f0e21fc 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,8 +6,9 @@ environment: install: # We need wheel installed to build wheels + - "%PYTHON%\\python.exe -m pip install -U pip" - "%PYTHON%\\python.exe -m pip install -U cython" - - "%PYTHON%\\Scripts\\cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx" + - "%PYTHON%\\Scripts\\cython --cplus msgpack/_cmsgpack.pyx" build: off diff --git a/docker/runtests.sh b/docker/runtests.sh index f6c2c68..c6bbf60 100755 --- a/docker/runtests.sh +++ b/docker/runtests.sh @@ -8,7 +8,7 @@ for V in cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do $PYBIN/pip install pytest pushd test # prevent importing msgpack package in current directory. $PYBIN/python -c 'import sys; print(hex(sys.maxsize))' - $PYBIN/python -c 'from msgpack import _msgpack' # Ensure extension is available + $PYBIN/python -c 'from msgpack import _cmsgpack' # Ensure extension is available $PYBIN/pytest -v . popd done diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 7c5d4c0..7493c4c 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -22,7 +22,7 @@ if os.environ.get('MSGPACK_PUREPYTHON'): from msgpack.fallback import Packer, unpackb, Unpacker else: try: - from msgpack._msgpack import Packer, unpackb, Unpacker + from msgpack._cmsgpack import Packer, unpackb, Unpacker except ImportError: from msgpack.fallback import Packer, unpackb, Unpacker diff --git a/msgpack/_cmsgpack.pyx b/msgpack/_cmsgpack.pyx new file mode 100644 index 0000000..a48d5b5 --- /dev/null +++ b/msgpack/_cmsgpack.pyx @@ -0,0 +1,4 @@ +# coding: utf-8 +#cython: embedsignature=True, c_string_encoding=ascii, language_level=2 +include "_packer.pyx" +include "_unpacker.pyx" diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx deleted file mode 100644 index 4381394..0000000 --- a/msgpack/_msgpack.pyx +++ /dev/null @@ -1,4 +0,0 @@ -# coding: utf-8 -#cython: embedsignature=True, c_string_encoding=ascii -include "_packer.pyx" -include "_unpacker.pyx" diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 2643f85..3be593f 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -1,9 +1,7 @@ # coding: utf-8 -#cython: embedsignature=True, c_string_encoding=ascii from cpython cimport * -from cpython.version cimport PY_MAJOR_VERSION -from cpython.exc cimport PyErr_WarnEx +from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact from msgpack import ExtType @@ -11,8 +9,6 @@ from msgpack import ExtType cdef extern from "Python.h": int PyMemoryView_Check(object obj) - int PyByteArray_Check(object obj) - int PyByteArray_CheckExact(object obj) char* PyUnicode_AsUTF8AndSize(object obj, Py_ssize_t *l) except NULL @@ -204,7 +200,7 @@ cdef class Packer(object): elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): L = len(o) if L > ITEM_LIMIT: - raise ValueError("%s is too large" % type(o).__name__) + PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) rawval = o ret = msgpack_pack_bin(&self.pk, L) if ret == 0: @@ -280,7 +276,7 @@ cdef class Packer(object): default_used = 1 continue else: - raise TypeError("can't serialize %r" % (o,)) + PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name) return ret cpdef pack(self, object obj): diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index e168587..aeebe2a 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -1,26 +1,6 @@ # coding: utf-8 -#cython: embedsignature=True, c_string_encoding=ascii -from cpython.version cimport PY_MAJOR_VERSION -from cpython.bytes cimport ( - PyBytes_AsString, - PyBytes_FromStringAndSize, - PyBytes_Size, -) -from cpython.buffer cimport ( - Py_buffer, - PyObject_CheckBuffer, - PyObject_GetBuffer, - PyBuffer_Release, - PyBuffer_IsContiguous, - PyBUF_READ, - PyBUF_SIMPLE, - PyBUF_FULL_RO, -) -from cpython.mem cimport PyMem_Malloc, PyMem_Free -from cpython.object cimport PyCallable_Check -from cpython.ref cimport Py_DECREF -from cpython.exc cimport PyErr_WarnEx +from cpython cimport * cdef extern from "Python.h": ctypedef struct PyObject diff --git a/setup.py b/setup.py index 8b8f7bd..eb9403f 100755 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ if len(version) > 3 and version[3] != 'final': if have_cython: class Sdist(sdist): def __init__(self, *args, **kwargs): - cythonize('msgpack/_msgpack.pyx') + cythonize('msgpack/_cmsgpack.pyx') sdist.__init__(self, *args, **kwargs) else: Sdist = sdist @@ -84,8 +84,8 @@ else: ext_modules = [] if not hasattr(sys, 'pypy_version_info'): - ext_modules.append(Extension('msgpack._msgpack', - sources=['msgpack/_msgpack.cpp'], + ext_modules.append(Extension('msgpack._cmsgpack', + sources=['msgpack/_cmsgpack.cpp'], libraries=libraries, include_dirs=['.'], define_macros=macros, From 2f808b6e012bdc506ca83a33c2f53af1b255a069 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 14 Nov 2018 20:04:22 +0900 Subject: [PATCH 0967/1172] Try language_level=3 (#329) --- msgpack/_cmsgpack.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_cmsgpack.pyx b/msgpack/_cmsgpack.pyx index a48d5b5..8ebdbf5 100644 --- a/msgpack/_cmsgpack.pyx +++ b/msgpack/_cmsgpack.pyx @@ -1,4 +1,4 @@ # coding: utf-8 -#cython: embedsignature=True, c_string_encoding=ascii, language_level=2 +#cython: embedsignature=True, c_string_encoding=ascii, language_level=3 include "_packer.pyx" include "_unpacker.pyx" From 8b6ce53cce40e528af7cce89f358f7dde1a09289 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 14 Nov 2018 21:06:16 +0900 Subject: [PATCH 0968/1172] s/iteritems/items/g (#330) --- msgpack/_packer.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 3be593f..bfde043 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -226,7 +226,7 @@ cdef class Packer(object): raise ValueError("dict is too large") ret = msgpack_pack_map(&self.pk, L) if ret == 0: - for k, v in d.iteritems(): + for k, v in d.items(): ret = self._pack(k, nest_limit-1) if ret != 0: break ret = self._pack(v, nest_limit-1) From 44254dd35e8aa3cfd6706e14effab117d7f22c25 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 20 Nov 2018 13:12:49 +0900 Subject: [PATCH 0969/1172] Add StackError and FormatError (#331) --- ChangeLog.rst | 22 ++++++++++++-------- Makefile | 3 ++- msgpack/_unpacker.pyx | 26 +++++++++++++++++++++-- msgpack/exceptions.py | 12 ++++++++++- msgpack/fallback.py | 43 ++++++++++++++++++++++++++++++++++----- msgpack/unpack_template.h | 29 +++++--------------------- test/test_except.py | 33 ++++++++++++++++++++++++------ 7 files changed, 120 insertions(+), 48 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index cc6b5e4..d39e76b 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -5,21 +5,25 @@ Release Date: TBD Important changes ------------------- - -Extension modules are merged. There is ``msgpack._msgpack`` instead of -``msgpack._packer`` and ``msgpack._unpacker``. (#314) - -unpacker: Default size limits is smaller than before to avoid DoS attack. -If you need to handle large data, you need to specify limits manually. +----------------- +* unpacker: Default size limits is smaller than before to avoid DoS attack. + If you need to handle large data, you need to specify limits manually. (#319) Other changes --------------- +------------- -Add ``Unpacker.getbuffer()`` method. +* Extension modules are merged. There is ``msgpack._msgpack`` instead of + ``msgpack._packer`` and ``msgpack._unpacker``. (#314) +* Add ``Unpacker.getbuffer()`` method. (#320) + +* unpacker: ``msgpack.StackError`` is raised when input data contains too + nested data. (#331) + +* unpacker: ``msgpack.FormatError`` is raised when input data is not valid + msgpack format. (#331) 0.5.6 diff --git a/Makefile b/Makefile index b65aa85..5828ed4 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,8 @@ cython: cython --cplus msgpack/_cmsgpack.pyx .PHONY: test -test: +test: cython + pip install -e . pytest -v test MSGPACK_PUREPYTHON=1 pytest -v test diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index aeebe2a..69330d3 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -16,6 +16,8 @@ from msgpack.exceptions import ( BufferFull, OutOfData, ExtraData, + FormatError, + StackError, ) from msgpack import ExtType @@ -149,7 +151,11 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, """ Unpack packed_bytes to object. Returns an unpacked object. - Raises `ValueError` when `packed` contains extra bytes. + Raises ``ExtraData`` when *packed* contains extra bytes. + Raises ``ValueError`` when *packed* is incomplete. + Raises ``FormatError`` when *packed* is not valid msgpack. + Raises ``StackError`` when *packed* contains too nested. + Other exceptions can be raised during unpacking. See :class:`Unpacker` for options. """ @@ -187,6 +193,12 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) return obj unpack_clear(&ctx) + if ret == 0: + raise ValueError("Unpack failed: incomplete input") + elif ret == -2: + raise FormatError + elif ret == -3: + raise StackError raise ValueError("Unpack failed: error = %d" % (ret,)) @@ -201,7 +213,7 @@ def unpack(object stream, **kwargs): cdef class Unpacker(object): """Streaming unpacker. - arguments: + Arguments: :param file_like: File-like object having `.read(n)` method. @@ -279,6 +291,12 @@ cdef class Unpacker(object): unpacker.feed(buf) for o in unpacker: process(o) + + Raises ``ExtraData`` when *packed* contains extra bytes. + Raises ``OutOfData`` when *packed* is incomplete. + Raises ``FormatError`` when *packed* is not valid msgpack. + Raises ``StackError`` when *packed* contains too nested. + Other exceptions can be raised during unpacking. """ cdef unpack_context ctx cdef char* buf @@ -451,6 +469,10 @@ cdef class Unpacker(object): raise StopIteration("No more data to unpack.") else: raise OutOfData("No more data to unpack.") + elif ret == -2: + raise FormatError + elif ret == -3: + raise StackError else: raise ValueError("Unpack failed: error = %d" % (ret,)) diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py index 5bee5b2..d6d2615 100644 --- a/msgpack/exceptions.py +++ b/msgpack/exceptions.py @@ -6,6 +6,7 @@ class UnpackException(Exception): Exception instead. """ + class BufferFull(UnpackException): pass @@ -14,6 +15,14 @@ class OutOfData(UnpackException): pass +class FormatError(ValueError, UnpackException): + """Invalid msgpack format""" + + +class StackError(ValueError, UnpackException): + """Too nested""" + + # Deprecated. Use ValueError instead UnpackValueError = ValueError @@ -24,6 +33,7 @@ class ExtraData(UnpackValueError): This exception is raised while only one-shot (not streaming) unpack. """ + def __init__(self, unpacked, extra): self.unpacked = unpacked self.extra = extra @@ -32,7 +42,7 @@ class ExtraData(UnpackValueError): return "unpack(b) received extra data." -#Deprecated. Use Exception instead to catch all exception during packing. +# Deprecated. Use Exception instead to catch all exception during packing. PackException = Exception PackValueError = ValueError PackOverflowError = OverflowError diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 04fb5b9..9c767a7 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -18,6 +18,16 @@ else: def dict_iteritems(d): return d.iteritems() +if sys.version_info < (3, 5): + # Ugly hack... + RecursionError = RuntimeError + + def _is_recursionerror(e): + return len(e.args) == 1 and isinstance(e.args[0], str) and \ + e.args[0].startswith('maximum recursion depth exceeded') +else: + def _is_recursionerror(e): + return True if hasattr(sys, 'pypy_version_info'): # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own @@ -52,7 +62,10 @@ else: from msgpack.exceptions import ( BufferFull, OutOfData, - ExtraData) + ExtraData, + FormatError, + StackError, +) from msgpack import ExtType @@ -109,7 +122,12 @@ def unpackb(packed, **kwargs): """ Unpack an object from `packed`. - Raises `ExtraData` when `packed` contains extra bytes. + Raises ``ExtraData`` when *packed* contains extra bytes. + Raises ``ValueError`` when *packed* is incomplete. + Raises ``FormatError`` when *packed* is not valid msgpack. + Raises ``StackError`` when *packed* contains too nested. + Other exceptions can be raised during unpacking. + See :class:`Unpacker` for options. """ unpacker = Unpacker(None, **kwargs) @@ -117,7 +135,11 @@ def unpackb(packed, **kwargs): try: ret = unpacker._unpack() except OutOfData: - raise ValueError("Data is not enough.") + raise ValueError("Unpack failed: incomplete input") + except RecursionError as e: + if _is_recursionerror(e): + raise StackError + raise if unpacker._got_extradata(): raise ExtraData(ret, unpacker._get_extradata()) return ret @@ -211,6 +233,12 @@ class Unpacker(object): unpacker.feed(buf) for o in unpacker: process(o) + + Raises ``ExtraData`` when *packed* contains extra bytes. + Raises ``OutOfData`` when *packed* is incomplete. + Raises ``FormatError`` when *packed* is not valid msgpack. + Raises ``StackError`` when *packed* contains too nested. + Other exceptions can be raised during unpacking. """ def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, @@ -561,7 +589,7 @@ class Unpacker(object): raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP else: - raise ValueError("Unknown header: 0x%x" % b) + raise FormatError("Unknown header: 0x%x" % b) return typ, n, obj def _unpack(self, execute=EX_CONSTRUCT): @@ -637,6 +665,8 @@ class Unpacker(object): except OutOfData: self._consume() raise StopIteration + except RecursionError: + raise StackError next = __next__ @@ -645,7 +675,10 @@ class Unpacker(object): self._consume() def unpack(self): - ret = self._unpack(EX_CONSTRUCT) + try: + ret = self._unpack(EX_CONSTRUCT) + except RecursionError: + raise StackError self._consume() return ret diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 525dea2..a78b7fa 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -123,7 +123,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize goto _fixed_trail_again #define start_container(func, count_, ct_) \ - if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \ + if(top >= MSGPACK_EMBED_STACK_SIZE) { ret = -3; goto _end; } \ if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ if((count_) == 0) { obj = stack[top].obj; \ if (construct_cb(func##_end)(user, &obj) < 0) { goto _failed; } \ @@ -132,27 +132,6 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize stack[top].size = count_; \ stack[top].count = 0; \ ++top; \ - /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ - /*printf("stack push %d\n", top);*/ \ - /* FIXME \ - if(top >= stack_size) { \ - if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \ - size_t csize = sizeof(unpack_stack) * MSGPACK_EMBED_STACK_SIZE; \ - size_t nsize = csize * 2; \ - unpack_stack* tmp = (unpack_stack*)malloc(nsize); \ - if(tmp == NULL) { goto _failed; } \ - memcpy(tmp, ctx->stack, csize); \ - ctx->stack = stack = tmp; \ - ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ - } else { \ - size_t nsize = sizeof(unpack_stack) * ctx->stack_size * 2; \ - unpack_stack* tmp = (unpack_stack*)realloc(ctx->stack, nsize); \ - if(tmp == NULL) { goto _failed; } \ - ctx->stack = stack = tmp; \ - ctx->stack_size = stack_size = stack_size * 2; \ - } \ - } \ - */ \ goto _header_again #define NEXT_CS(p) ((unsigned int)*p & 0x1f) @@ -229,7 +208,8 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize case 0xdf: // map 32 again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); default: - goto _failed; + ret = -2; + goto _end; } SWITCH_RANGE(0xa0, 0xbf) // FixRaw again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); @@ -239,7 +219,8 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); SWITCH_RANGE_DEFAULT - goto _failed; + ret = -2; + goto _end; SWITCH_RANGE_END // end CS_HEADER diff --git a/test/test_except.py b/test/test_except.py index 361d4ea..626c8be 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -2,7 +2,7 @@ # coding: utf-8 from pytest import raises -from msgpack import packb, unpackb +from msgpack import packb, unpackb, Unpacker, FormatError, StackError, OutOfData import datetime @@ -19,13 +19,34 @@ def test_raise_on_find_unsupported_value(): def test_raise_from_object_hook(): def hook(obj): raise DummyException + raises(DummyException, unpackb, packb({}), object_hook=hook) - raises(DummyException, unpackb, packb({'fizz': 'buzz'}), object_hook=hook) - raises(DummyException, unpackb, packb({'fizz': 'buzz'}), object_pairs_hook=hook) - raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), object_hook=hook) - raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), object_pairs_hook=hook) + raises(DummyException, unpackb, packb({"fizz": "buzz"}), object_hook=hook) + raises(DummyException, unpackb, packb({"fizz": "buzz"}), object_pairs_hook=hook) + raises(DummyException, unpackb, packb({"fizz": {"buzz": "spam"}}), object_hook=hook) + raises( + DummyException, + unpackb, + packb({"fizz": {"buzz": "spam"}}), + object_pairs_hook=hook, + ) def test_invalidvalue(): + incomplete = b"\xd9\x97#DL_" # raw8 - length=0x97 with raises(ValueError): - unpackb(b'\xd9\x97#DL_') + unpackb(incomplete) + + with raises(OutOfData): + unpacker = Unpacker() + unpacker.feed(incomplete) + unpacker.unpack() + + with raises(FormatError): + unpackb(b"\xc1") # (undefined tag) + + with raises(FormatError): + unpackb(b"\x91\xc1") # fixarray(len=1) [ (undefined tag) ] + + with raises(StackError): + unpackb(b"\x91" * 3000) # nested fixarray(len=1) From ab2415eaa0cdbe8e5b6e248d447cf5e66e858eb2 Mon Sep 17 00:00:00 2001 From: jkorvin Date: Tue, 20 Nov 2018 09:24:35 +0300 Subject: [PATCH 0970/1172] Unpacker: allow to use buffer with size greater than 2 GB (#332) --- msgpack/_unpacker.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 69330d3..a5403d8 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -320,7 +320,7 @@ cdef class Unpacker(object): def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=True, bint raw=True, object object_hook=None, object object_pairs_hook=None, object list_hook=None, - encoding=None, unicode_errors=None, int max_buffer_size=0, + encoding=None, unicode_errors=None, Py_ssize_t max_buffer_size=0, object ext_hook=ExtType, Py_ssize_t max_str_len=1024*1024, Py_ssize_t max_bin_len=1024*1024, From 3c9c6edbc88908fceb3c69ff3d6455be8b5914c8 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 20 Nov 2018 15:48:44 +0900 Subject: [PATCH 0971/1172] Update README --- ChangeLog.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index d39e76b..8bc80e6 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -7,15 +7,21 @@ Release Date: TBD Important changes ----------------- -* unpacker: Default size limits is smaller than before to avoid DoS attack. +* unpacker: Default value of input limits are smaller than before to avoid DoS attack. If you need to handle large data, you need to specify limits manually. (#319) +* Unpacker doesn't wrap underlaying ``ValueError`` (including ``UnicodeError``) into + ``UnpackValueError``. If you want to catch all exception during unpack, you need + to use ``try ... except Exception`` with minimum try code block. (#323, #233) + +* ``PackValueError`` and ``PackOverflowError`` are also removed. You need to catch + normal ``ValueError`` and ``OverflowError``. (#323, #233) Other changes ------------- -* Extension modules are merged. There is ``msgpack._msgpack`` instead of - ``msgpack._packer`` and ``msgpack._unpacker``. (#314) +* Extension modules are merged. There is ``msgpack._cmsgpack`` instead of + ``msgpack._packer`` and ``msgpack._unpacker``. (#314, #328) * Add ``Unpacker.getbuffer()`` method. (#320) From e9086a34e4b3d64df78314339f152c800e79c8e1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 29 Nov 2018 22:29:38 +0900 Subject: [PATCH 0972/1172] Add strict_map_key option to unpacker --- msgpack/_unpacker.pyx | 17 ++++++++++++----- msgpack/unpack.h | 5 +++++ test/test_except.py | 11 +++++++++++ 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index a5403d8..2163425 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -27,6 +27,7 @@ cdef extern from "unpack.h": bint use_list bint raw bint has_pairs_hook # call object_hook with k-v pairs + bint strict_map_key PyObject* object_hook PyObject* list_hook PyObject* ext_hook @@ -56,7 +57,7 @@ cdef extern from "unpack.h": cdef inline init_ctx(unpack_context *ctx, object object_hook, object object_pairs_hook, object list_hook, object ext_hook, - bint use_list, bint raw, + bint use_list, bint raw, bint strict_map_key, const char* encoding, const char* unicode_errors, Py_ssize_t max_str_len, Py_ssize_t max_bin_len, Py_ssize_t max_array_len, Py_ssize_t max_map_len, @@ -64,6 +65,7 @@ cdef inline init_ctx(unpack_context *ctx, unpack_init(ctx) ctx.user.use_list = use_list ctx.user.raw = raw + ctx.user.strict_map_key = strict_map_key ctx.user.object_hook = ctx.user.list_hook = NULL ctx.user.max_str_len = max_str_len ctx.user.max_bin_len = max_bin_len @@ -140,7 +142,7 @@ cdef inline int get_data_from_buffer(object obj, return 1 def unpackb(object packed, object object_hook=None, object list_hook=None, - bint use_list=True, bint raw=True, + bint use_list=True, bint raw=True, bint strict_map_key=False, encoding=None, unicode_errors=None, object_pairs_hook=None, ext_hook=ExtType, Py_ssize_t max_str_len=1024*1024, @@ -180,7 +182,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol) try: init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook, - use_list, raw, cenc, cerr, + use_list, raw, strict_map_key, cenc, cerr, max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) ret = unpack_construct(&ctx, buf, buf_len, &off) finally: @@ -236,6 +238,11 @@ cdef class Unpacker(object): *encoding* option which is deprecated overrides this option. + :param bool strict_map_key: + If true, only str or bytes are accepted for map (dict) keys. + It's False by default for backward-compatibility. + But it will be True from msgpack 1.0. + :param callable object_hook: When specified, it should be callable. Unpacker calls it with a dict argument after unpacking msgpack map. @@ -318,7 +325,7 @@ cdef class Unpacker(object): self.buf = NULL def __init__(self, file_like=None, Py_ssize_t read_size=0, - bint use_list=True, bint raw=True, + bint use_list=True, bint raw=True, bint strict_map_key=False, object object_hook=None, object object_pairs_hook=None, object list_hook=None, encoding=None, unicode_errors=None, Py_ssize_t max_buffer_size=0, object ext_hook=ExtType, @@ -366,7 +373,7 @@ cdef class Unpacker(object): cerr = unicode_errors init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, - ext_hook, use_list, raw, cenc, cerr, + ext_hook, use_list, raw, strict_map_key, cenc, cerr, max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 63e5543..85dbbed 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -23,6 +23,7 @@ typedef struct unpack_user { bool use_list; bool raw; bool has_pairs_hook; + bool strict_map_key; PyObject *object_hook; PyObject *list_hook; PyObject *ext_hook; @@ -188,6 +189,10 @@ static inline int unpack_callback_map(unpack_user* u, unsigned int n, msgpack_un static inline int unpack_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { + if (u->strict_map_key && !PyUnicode_CheckExact(k) && !PyBytes_CheckExact(k)) { + PyErr_Format(PyExc_ValueError, "%.100s is not allowed for map key", Py_TYPE(k)->tp_name); + return -1; + } if (u->has_pairs_hook) { msgpack_unpack_object item = PyTuple_Pack(2, k, v); if (!item) diff --git a/test/test_except.py b/test/test_except.py index 626c8be..40ca3ee 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -50,3 +50,14 @@ def test_invalidvalue(): with raises(StackError): unpackb(b"\x91" * 3000) # nested fixarray(len=1) + + +def test_strict_map_key(): + valid = {u"unicode": 1, b"bytes": 2} + packed = packb(valid, use_bin_type=True) + assert valid == unpackb(packed, raw=True) + + invalid = {42: 1} + packed = packb(invalid, use_bin_type=True) + with raises(ValueError): + unpackb(packed, raw=True) From dc1b9930793ea57aa4e2a9773a23582b5483c53a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 29 Nov 2018 22:35:12 +0900 Subject: [PATCH 0973/1172] Implement strict_map_key to fallback unpacker. --- msgpack/fallback.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 9c767a7..be60cca 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -179,6 +179,11 @@ class Unpacker(object): *encoding* option which is deprecated overrides this option. + :param bool strict_map_key: + If true, only str or bytes are accepted for map (dict) keys. + It's False by default for backward-compatibility. + But it will be True from msgpack 1.0. + :param callable object_hook: When specified, it should be callable. Unpacker calls it with a dict argument after unpacking msgpack map. @@ -241,7 +246,7 @@ class Unpacker(object): Other exceptions can be raised during unpacking. """ - def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, + def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, strict_map_key=False, object_hook=None, object_pairs_hook=None, list_hook=None, encoding=None, unicode_errors=None, max_buffer_size=0, ext_hook=ExtType, @@ -286,6 +291,7 @@ class Unpacker(object): raise ValueError("read_size must be smaller than max_buffer_size") self._read_size = read_size or min(self._max_buffer_size, 16*1024) self._raw = bool(raw) + self._strict_map_key = bool(strict_map_key) self._encoding = encoding self._unicode_errors = unicode_errors self._use_list = use_list @@ -633,6 +639,8 @@ class Unpacker(object): ret = {} for _ in xrange(n): key = self._unpack(EX_CONSTRUCT) + if self._strict_map_key and type(key) not in (unicode, bytes): + raise ValueError("%s is not allowed for map key" % str(type(key))) ret[key] = self._unpack(EX_CONSTRUCT) if self._object_hook is not None: ret = self._object_hook(ret) From e76091a82c04d092a363b0e1f7307dc70bf784f5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 29 Nov 2018 22:38:22 +0900 Subject: [PATCH 0974/1172] Fix test --- test/test_except.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_except.py b/test/test_except.py index 40ca3ee..01961d2 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -55,9 +55,9 @@ def test_invalidvalue(): def test_strict_map_key(): valid = {u"unicode": 1, b"bytes": 2} packed = packb(valid, use_bin_type=True) - assert valid == unpackb(packed, raw=True) + assert valid == unpackb(packed, raw=True, strict_map_key=True) invalid = {42: 1} packed = packb(invalid, use_bin_type=True) with raises(ValueError): - unpackb(packed, raw=True) + unpackb(packed, raw=True, strict_map_key=True) From ab789813b8e5786f404e9132ff1c6f2647dc4afa Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 30 Nov 2018 11:36:15 +0900 Subject: [PATCH 0975/1172] Fix test --- test/test_except.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_except.py b/test/test_except.py index 01961d2..5544f2b 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -55,9 +55,9 @@ def test_invalidvalue(): def test_strict_map_key(): valid = {u"unicode": 1, b"bytes": 2} packed = packb(valid, use_bin_type=True) - assert valid == unpackb(packed, raw=True, strict_map_key=True) + assert valid == unpackb(packed, raw=False, strict_map_key=True) invalid = {42: 1} packed = packb(invalid, use_bin_type=True) with raises(ValueError): - unpackb(packed, raw=True, strict_map_key=True) + unpackb(packed, raw=False, strict_map_key=True) From 8ae6320072e746fad29bc14a095569811e009695 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 30 Nov 2018 11:42:51 +0900 Subject: [PATCH 0976/1172] Fix fallback --- msgpack/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index be60cca..ae2fcfc 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -639,7 +639,7 @@ class Unpacker(object): ret = {} for _ in xrange(n): key = self._unpack(EX_CONSTRUCT) - if self._strict_map_key and type(key) not in (unicode, bytes): + if self._strict_map_key and type(key) not in (Unicode, bytes): raise ValueError("%s is not allowed for map key" % str(type(key))) ret[key] = self._unpack(EX_CONSTRUCT) if self._object_hook is not None: From 04cf8fc7f4b9e8dd32d809cd2c45b05b83d7f913 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 30 Nov 2018 14:04:18 +0900 Subject: [PATCH 0977/1172] Update ChangeLog --- ChangeLog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 8bc80e6..806007e 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -17,6 +17,11 @@ Important changes * ``PackValueError`` and ``PackOverflowError`` are also removed. You need to catch normal ``ValueError`` and ``OverflowError``. (#323, #233) +* Unpacker has ``strict_map_key`` option now. When it is true, only bytes and str + (unicode in Python 2) are allowed for map keys. It is recommended to avoid + hashdos. Default value of this option is False for backward compatibility reason. + But it will be changed True in 1.0. (#296, #334) + Other changes ------------- From 93b5953eae11fa3a8668de16a3ccbf20d7bf0fd9 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 30 Nov 2018 16:05:31 +0900 Subject: [PATCH 0978/1172] Update tox.ini --- tox.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 68a2f53..0945a6d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py35,py36}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86 +envlist = {py27,py35,py36,py37}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86 [variants:pure] setenv= @@ -11,7 +11,7 @@ deps= changedir=test commands= - c,x86: python -c 'from msgpack import _packer, _unpacker' + c,x86: python -c 'from msgpack import _cmsgpack' c,x86: py.test pure: py.test @@ -23,7 +23,7 @@ deps= changedir=test commands= python -c 'import sys; print(hex(sys.maxsize))' - python -c 'from msgpack import _packer, _unpacker' + python -c 'from msgpack import _cmsgpack' py.test [testenv:py34-x86] @@ -34,5 +34,5 @@ deps= changedir=test commands= python -c 'import sys; print(hex(sys.maxsize))' - python -c 'from msgpack import _packer, _unpacker' + python -c 'from msgpack import _cmsgpack' py.test From bbdfd4d92e54e89604a2ebf6af86ced4ae5ae05d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 30 Nov 2018 16:28:41 +0900 Subject: [PATCH 0979/1172] cleanup --- msgpack/fallback.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index ae2fcfc..57b436a 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -4,19 +4,19 @@ import sys import struct import warnings -if sys.version_info[0] == 3: - PY3 = True + +if sys.version_info[0] == 2: + PY2 = True + int_types = (int, long) + def dict_iteritems(d): + return d.iteritems() +else: + PY2 = False int_types = int - Unicode = str + unicode = str xrange = range def dict_iteritems(d): return d.items() -else: - PY3 = False - int_types = (int, long) - Unicode = unicode - def dict_iteritems(d): - return d.iteritems() if sys.version_info < (3, 5): # Ugly hack... @@ -97,7 +97,7 @@ def _get_data_from_buffer(obj): view = memoryview(obj) except TypeError: # try to use legacy buffer protocol if 2.7, otherwise re-raise - if not PY3: + if PY2: view = memoryview(buffer(obj)) warnings.warn("using old buffer interface to unpack %s; " "this leads to unpacking errors if slicing is used and " @@ -639,7 +639,7 @@ class Unpacker(object): ret = {} for _ in xrange(n): key = self._unpack(EX_CONSTRUCT) - if self._strict_map_key and type(key) not in (Unicode, bytes): + if self._strict_map_key and type(key) not in (unicode, bytes): raise ValueError("%s is not allowed for map key" % str(type(key))) ret[key] = self._unpack(EX_CONSTRUCT) if self._object_hook is not None: @@ -819,7 +819,7 @@ class Packer(object): raise ValueError("%s is too large" % type(obj).__name__) self._pack_bin_header(n) return self._buffer.write(obj) - if check(obj, Unicode): + if check(obj, unicode): if self._encoding is None: raise TypeError( "Can't encode unicode string: " @@ -1006,7 +1006,7 @@ class Packer(object): def getbuffer(self): """Return view of internal buffer.""" - if USING_STRINGBUILDER or not PY3: + if USING_STRINGBUILDER or PY2: return memoryview(self.bytes()) else: return self._buffer.getbuffer() From cc7fd5722b779d438f7d226a9c7f61115764b39c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 30 Nov 2018 19:03:44 +0900 Subject: [PATCH 0980/1172] 0.6.0 --- ChangeLog.rst | 3 ++- msgpack/_version.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 806007e..651ba62 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,8 +1,9 @@ 0.6.0 ====== -Release Date: TBD +Release Date: 2018-11-30 +This release contains some backward incompatible changes for security reason (DoS). Important changes ----------------- diff --git a/msgpack/_version.py b/msgpack/_version.py index 0952ec6..49a4103 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 6, 0, 'dev') +version = (0, 6, 0) From b1d658e7a0aed4e723f9b6b238cc9f2f876e54d7 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 30 Nov 2018 19:25:14 +0900 Subject: [PATCH 0981/1172] AppVeyor: Add Python 3.7 and remove 3.6 --- appveyor.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index f0e21fc..bd0800a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -21,14 +21,14 @@ test_script: - ci\\runtests.bat - set PYTHON="C:\\Python27-x64" - ci\\runtests.bat - - set PYTHON="C:\\Python35" - - ci\\runtests.bat - - set PYTHON="C:\\Python35-x64" - - ci\\runtests.bat - set PYTHON="C:\\Python36" - ci\\runtests.bat - set PYTHON="C:\\Python36-x64" - ci\\runtests.bat + - set PYTHON="C:\\Python37" + - ci\\runtests.bat + - set PYTHON="C:\\Python37-x64" + - ci\\runtests.bat after_test: # This step builds your wheels. From b8bf3c950c4474d3af82a6b6bda6326b2e197a5e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 4 Dec 2018 17:18:34 +0900 Subject: [PATCH 0982/1172] Build linux wheel for Python 3.7 --- docker/buildwheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh index f586a8d..d8c74cb 100644 --- a/docker/buildwheel.sh +++ b/docker/buildwheel.sh @@ -4,7 +4,7 @@ set -e -x ARCH=`uname -p` echo "arch=$ARCH" -for V in cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do +for V in cp37-cp37m cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do PYBIN=/opt/python/$V/bin rm -rf build/ # Avoid lib build by narrow Python is used by wide python $PYBIN/python setup.py bdist_wheel -p manylinux1_${ARCH} From 197e30723a242e3ba31f9634b3263ae4cb2937b1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 4 Dec 2018 20:10:21 +0900 Subject: [PATCH 0983/1172] Fix docstring --- msgpack/_unpacker.pyx | 2 +- msgpack/fallback.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 2163425..4ea0545 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -271,7 +271,7 @@ cdef class Unpacker(object): Limits max length of map. (default: 32*1024) :param int max_ext_len: - Limits max length of map. (default: 1024*1024) + Limits max size of ext type. (default: 1024*1024) :param str encoding: Deprecated, use raw instead. diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 57b436a..4567e2d 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -220,7 +220,7 @@ class Unpacker(object): Limits max length of map. (default: 32*1024) :param int max_ext_len: - Limits max length of map. (default: 1024*1024) + Limits max size of ext type. (default: 1024*1024) example of streaming deserialize from file-like object:: From f46523b1af7ff2d408da8500ea36a4f9f2abe915 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 7 Jan 2019 21:10:40 +0900 Subject: [PATCH 0984/1172] use _PyFloat APIs to (de)serialize (#340) --- msgpack/pack_template.h | 13 +++---------- msgpack/unpack_template.h | 14 ++++---------- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 5d1088f..69982f4 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -566,24 +566,17 @@ if(sizeof(unsigned long long) == 2) { static inline int msgpack_pack_float(msgpack_packer* x, float d) { - union { float f; uint32_t i; } mem; - mem.f = d; unsigned char buf[5]; - buf[0] = 0xca; _msgpack_store32(&buf[1], mem.i); + buf[0] = 0xca; + _PyFloat_Pack4(d, &buf[1], 0); msgpack_pack_append_buffer(x, buf, 5); } static inline int msgpack_pack_double(msgpack_packer* x, double d) { - union { double f; uint64_t i; } mem; - mem.f = d; unsigned char buf[9]; buf[0] = 0xcb; -#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi - // https://github.com/msgpack/msgpack-perl/pull/1 - mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL); -#endif - _msgpack_store64(&buf[1], mem.i); + _PyFloat_Pack8(d, &buf[1], 0); msgpack_pack_append_buffer(x, buf, 9); } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index a78b7fa..9924b9c 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -243,17 +243,11 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize _msgpack_load32(uint32_t,n)+1, _ext_zero); case CS_FLOAT: { - union { uint32_t i; float f; } mem; - mem.i = _msgpack_load32(uint32_t,n); - push_fixed_value(_float, mem.f); } + double f = _PyFloat_Unpack4((unsigned char*)n, 0); + push_fixed_value(_float, f); } case CS_DOUBLE: { - union { uint64_t i; double f; } mem; - mem.i = _msgpack_load64(uint64_t,n); -#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi - // https://github.com/msgpack/msgpack-perl/pull/1 - mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL); -#endif - push_fixed_value(_double, mem.f); } + double f = _PyFloat_Unpack8((unsigned char*)n, 0); + push_fixed_value(_double, f); } case CS_UINT_8: push_fixed_value(_uint8, *(uint8_t*)n); case CS_UINT_16: From 28b5f46a34933cc177aca333203d1344b5e3639a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 24 Jan 2019 18:46:39 +0900 Subject: [PATCH 0985/1172] Auto limit configuration (#342) --- msgpack/_unpacker.pyx | 56 +++++++++++++++++++++++++++++++------------ msgpack/fallback.py | 39 ++++++++++++++++++++---------- test/test_limits.py | 25 ++++++++++++++++++- 3 files changed, 92 insertions(+), 28 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 4ea0545..38119c0 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -145,11 +145,11 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=True, bint raw=True, bint strict_map_key=False, encoding=None, unicode_errors=None, object_pairs_hook=None, ext_hook=ExtType, - Py_ssize_t max_str_len=1024*1024, - Py_ssize_t max_bin_len=1024*1024, - Py_ssize_t max_array_len=128*1024, - Py_ssize_t max_map_len=32*1024, - Py_ssize_t max_ext_len=1024*1024): + Py_ssize_t max_str_len=-1, + Py_ssize_t max_bin_len=-1, + Py_ssize_t max_array_len=-1, + Py_ssize_t max_map_len=-1, + Py_ssize_t max_ext_len=-1): """ Unpack packed_bytes to object. Returns an unpacked object. @@ -160,6 +160,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, Other exceptions can be raised during unpacking. See :class:`Unpacker` for options. + + *max_xxx_len* options are configured automatically from ``len(packed)``. """ cdef unpack_context ctx cdef Py_ssize_t off = 0 @@ -180,6 +182,18 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cerr = unicode_errors get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol) + + if max_str_len == -1: + max_str_len = buf_len + if max_bin_len == -1: + max_bin_len = buf_len + if max_array_len == -1: + max_array_len = buf_len + if max_map_len == -1: + max_map_len = buf_len//2 + if max_ext_len == -1: + max_ext_len = buf_len + try: init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook, use_list, raw, strict_map_key, cenc, cerr, @@ -259,19 +273,19 @@ cdef class Unpacker(object): You should set this parameter when unpacking data from untrusted source. :param int max_str_len: - Limits max length of str. (default: 1024*1024) + Limits max length of str. (default: max_buffer_size or 1024*1024) :param int max_bin_len: - Limits max length of bin. (default: 1024*1024) + Limits max length of bin. (default: max_buffer_size or 1024*1024) :param int max_array_len: - Limits max length of array. (default: 128*1024) + Limits max length of array. (default: max_buffer_size or 128*1024) :param int max_map_len: - Limits max length of map. (default: 32*1024) + Limits max length of map. (default: max_buffer_size//2 or 32*1024) :param int max_ext_len: - Limits max size of ext type. (default: 1024*1024) + Limits max size of ext type. (default: max_buffer_size or 1024*1024) :param str encoding: Deprecated, use raw instead. @@ -329,11 +343,11 @@ cdef class Unpacker(object): object object_hook=None, object object_pairs_hook=None, object list_hook=None, encoding=None, unicode_errors=None, Py_ssize_t max_buffer_size=0, object ext_hook=ExtType, - Py_ssize_t max_str_len=1024*1024, - Py_ssize_t max_bin_len=1024*1024, - Py_ssize_t max_array_len=128*1024, - Py_ssize_t max_map_len=32*1024, - Py_ssize_t max_ext_len=1024*1024): + Py_ssize_t max_str_len=-1, + Py_ssize_t max_bin_len=-1, + Py_ssize_t max_array_len=-1, + Py_ssize_t max_map_len=-1, + Py_ssize_t max_ext_len=-1): cdef const char *cenc=NULL, cdef const char *cerr=NULL @@ -347,6 +361,18 @@ cdef class Unpacker(object): self.file_like_read = file_like.read if not PyCallable_Check(self.file_like_read): raise TypeError("`file_like.read` must be a callable.") + + if max_str_len == -1: + max_str_len = max_buffer_size or 1024*1024 + if max_bin_len == -1: + max_bin_len = max_buffer_size or 1024*1024 + if max_array_len == -1: + max_array_len = max_buffer_size or 128*1024 + if max_map_len == -1: + max_map_len = max_buffer_size//2 or 32*1024 + if max_ext_len == -1: + max_ext_len = max_buffer_size or 1024*1024 + if not max_buffer_size: max_buffer_size = INT_MAX if read_size > max_buffer_size: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 4567e2d..7524448 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -130,7 +130,7 @@ def unpackb(packed, **kwargs): See :class:`Unpacker` for options. """ - unpacker = Unpacker(None, **kwargs) + unpacker = Unpacker(None, max_buffer_size=len(packed), **kwargs) unpacker.feed(packed) try: ret = unpacker._unpack() @@ -208,19 +208,24 @@ class Unpacker(object): You should set this parameter when unpacking data from untrusted source. :param int max_str_len: - Limits max length of str. (default: 1024*1024) + (deprecated) Limits max length of str. + (default: max_buffer_size or 1024*1024) :param int max_bin_len: - Limits max length of bin. (default: 1024*1024) + (deprecated) Limits max length of bin. + (default: max_buffer_size or 1024*1024) :param int max_array_len: - Limits max length of array. (default: 128*1024) + Limits max length of array. + (default: max_buffer_size or 128*1024) :param int max_map_len: - Limits max length of map. (default: 32*1024) + Limits max length of map. + (default: max_buffer_size//2 or 32*1024) :param int max_ext_len: - Limits max size of ext type. (default: 1024*1024) + (deprecated) Limits max size of ext type. + (default: max_buffer_size or 1024*1024) example of streaming deserialize from file-like object:: @@ -250,12 +255,11 @@ class Unpacker(object): object_hook=None, object_pairs_hook=None, list_hook=None, encoding=None, unicode_errors=None, max_buffer_size=0, ext_hook=ExtType, - max_str_len=1024*1024, - max_bin_len=1024*1024, - max_array_len=128*1024, - max_map_len=32*1024, - max_ext_len=1024*1024): - + max_str_len=-1, + max_bin_len=-1, + max_array_len=-1, + max_map_len=-1, + max_ext_len=-1): if encoding is not None: warnings.warn( "encoding is deprecated, Use raw=False instead.", @@ -286,6 +290,17 @@ class Unpacker(object): # state, which _buf_checkpoint records. self._buf_checkpoint = 0 + if max_str_len == -1: + max_str_len = max_buffer_size or 1024*1024 + if max_bin_len == -1: + max_bin_len = max_buffer_size or 1024*1024 + if max_array_len == -1: + max_array_len = max_buffer_size or 128*1024 + if max_map_len == -1: + max_map_len = max_buffer_size//2 or 32*1024 + if max_ext_len == -1: + max_ext_len = max_buffer_size or 1024*1024 + self._max_buffer_size = max_buffer_size or 2**31-1 if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") diff --git a/test/test_limits.py b/test/test_limits.py index 74e48c1..8c7606f 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -105,7 +105,6 @@ def test_max_ext_len(): unpacker.unpack() - # PyPy fails following tests because of constant folding? # https://bugs.pypy.org/issue1721 #@pytest.mark.skipif(True, reason="Requires very large memory.") @@ -134,3 +133,27 @@ def test_max_ext_len(): # x.append(0) # with pytest.raises(ValueError): # packb(x) + + +# auto max len + +def test_auto_max_array_len(): + packed = b'\xde\x00\x06zz' + with pytest.raises(UnpackValueError): + unpackb(packed, raw=False) + + unpacker = Unpacker(max_buffer_size=5, raw=False) + unpacker.feed(packed) + with pytest.raises(UnpackValueError): + unpacker.unpack() + +def test_auto_max_map_len(): + # len(packed) == 6 -> max_map_len == 3 + packed = b'\xde\x00\x04zzz' + with pytest.raises(UnpackValueError): + unpackb(packed, raw=False) + + unpacker = Unpacker(max_buffer_size=6, raw=False) + unpacker.feed(packed) + with pytest.raises(UnpackValueError): + unpacker.unpack() From 464fe277e1165a5870d4edc040be9c9ac1c1df0c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 25 Jan 2019 20:52:57 +0900 Subject: [PATCH 0986/1172] Remove pytest warnings --- test/test_buffer.py | 5 ++++- test/test_extension.py | 10 ++++++++-- test/test_pack.py | 23 +++++++++++++++-------- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/test/test_buffer.py b/test/test_buffer.py index 87f359f..d723e8d 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -7,7 +7,10 @@ from msgpack import packb, unpackb def test_unpack_buffer(): from array import array buf = array('b') - buf.fromstring(packb((b'foo', b'bar'))) + try: + buf.frombytes(packb((b'foo', b'bar'))) + except AttributeError: # PY2 + buf.fromstring(packb((b'foo', b'bar'))) obj = unpackb(buf, use_list=1) assert [b'foo', b'bar'] == obj diff --git a/test/test_extension.py b/test/test_extension.py index d05d7ab..8aa0cbb 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -40,7 +40,10 @@ def test_extension_type(): print('default called', obj) if isinstance(obj, array.array): typecode = 123 # application specific typecode - data = obj.tostring() + try: + data = obj.tobytes() + except AttributeError: + data = obj.tostring() return ExtType(typecode, data) raise TypeError("Unknown type object %r" % (obj,)) @@ -48,7 +51,10 @@ def test_extension_type(): print('ext_hook called', code, data) assert code == 123 obj = array.array('d') - obj.fromstring(data) + try: + obj.frombytes(data) + except AttributeError: # PY2 + obj.fromstring(data) return obj obj = [42, b'hello', array.array('d', [1.1, 2.2, 3.3])] diff --git a/test/test_pack.py b/test/test_pack.py index 4608083..3658a97 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -2,13 +2,15 @@ # coding: utf-8 from __future__ import absolute_import, division, print_function, unicode_literals +from collections import OrderedDict +from io import BytesIO import struct + +import pytest from pytest import raises, xfail from msgpack import packb, unpackb, Unpacker, Packer, pack -from collections import OrderedDict -from io import BytesIO def check(data, use_list=False): re = unpackb(packb(data), use_list=use_list) @@ -47,7 +49,8 @@ def testPackUTF32(): # deprecated "РуÑÑкий текÑÑ‚", ] for td in test_data: - re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32') + with pytest.deprecated_call(): + re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32') assert re == td except LookupError as e: xfail(e) @@ -67,19 +70,23 @@ def testPackByteArrays(): check(td) def testIgnoreUnicodeErrors(): # deprecated - re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1) + with pytest.deprecated_call(): + re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1) assert re == "abcdef" def testStrictUnicodeUnpack(): - with raises(UnicodeDecodeError): - unpackb(packb(b'abc\xeddef'), raw=False, use_list=1) + packed = packb(b'abc\xeddef') + with pytest.raises(UnicodeDecodeError): + unpackb(packed, raw=False, use_list=1) def testStrictUnicodePack(): # deprecated with raises(UnicodeEncodeError): - packb("abc\xeddef", encoding='ascii', unicode_errors='strict') + with pytest.deprecated_call(): + packb("abc\xeddef", encoding='ascii', unicode_errors='strict') def testIgnoreErrorsPack(): # deprecated - re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw=False, use_list=1) + with pytest.deprecated_call(): + re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw=False, use_list=1) assert re == "abcdef" def testDecodeBinary(): From 9951b894555e4f9c7120375028e686f7420de92a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 25 Jan 2019 21:04:14 +0900 Subject: [PATCH 0987/1172] travis: Install new pytest --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1adbdc2..51917c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ matrix: install: - pip install -U pip - - pip install cython + - pip install -U cython pytest - make cython - pip install -e . From 280308e8ced50322414fd4f7426d56093a57dbf1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 25 Jan 2019 21:27:46 +0900 Subject: [PATCH 0988/1172] Recommend max_buffer_len instead of max_(str|bin|ext)_len --- ChangeLog.rst | 18 ++++++++++++++++++ msgpack/_unpacker.pyx | 9 ++++++--- msgpack/fallback.py | 20 ++++++++++---------- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 651ba62..2c988db 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,21 @@ +0.6.1 +====== + +Release Date: 2019-01-25 + +This release is for mitigating pain caused by v0.6.1 reduced max input limits +for security reason. + +* ``unpackb(data)`` configures ``max_*_len`` options from ``len(data)``, + instead of static default sizes. + +* ``Unpacker(max_buffer_len=N)`` configures ``max_*_len`` options from ``N``, + instead of static default sizes. + +* ``max_bin_len``, ``max_str_len``, and ``max_ext_len`` are deprecated. + Since this is minor release, it's document only deprecation. + + 0.6.0 ====== diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 38119c0..3c6d59e 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -273,9 +273,11 @@ cdef class Unpacker(object): You should set this parameter when unpacking data from untrusted source. :param int max_str_len: + Deprecated, use *max_buffer_size* instead. Limits max length of str. (default: max_buffer_size or 1024*1024) :param int max_bin_len: + Deprecated, use *max_buffer_size* instead. Limits max length of bin. (default: max_buffer_size or 1024*1024) :param int max_array_len: @@ -285,10 +287,11 @@ cdef class Unpacker(object): Limits max length of map. (default: max_buffer_size//2 or 32*1024) :param int max_ext_len: + Deprecated, use *max_buffer_size* instead. Limits max size of ext type. (default: max_buffer_size or 1024*1024) :param str encoding: - Deprecated, use raw instead. + Deprecated, use ``raw=False`` instead. Encoding used for decoding msgpack raw. If it is None (default), msgpack raw is deserialized to Python bytes. @@ -298,13 +301,13 @@ cdef class Unpacker(object): Example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like, raw=False) + unpacker = Unpacker(file_like, raw=False, max_buffer_size=10*1024*1024) for o in unpacker: process(o) Example of streaming deserialize from socket:: - unpacker = Unpacker(raw=False) + unpacker = Unpacker(raw=False, max_buffer_size=10*1024*1024) while True: buf = sock.recv(1024**2) if not buf: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 7524448..1aa3bdf 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -208,12 +208,12 @@ class Unpacker(object): You should set this parameter when unpacking data from untrusted source. :param int max_str_len: - (deprecated) Limits max length of str. - (default: max_buffer_size or 1024*1024) + Deprecated, use *max_buffer_size* instead. + Limits max length of str. (default: max_buffer_size or 1024*1024) :param int max_bin_len: - (deprecated) Limits max length of bin. - (default: max_buffer_size or 1024*1024) + Deprecated, use *max_buffer_size* instead. + Limits max length of bin. (default: max_buffer_size or 1024*1024) :param int max_array_len: Limits max length of array. @@ -224,18 +224,18 @@ class Unpacker(object): (default: max_buffer_size//2 or 32*1024) :param int max_ext_len: - (deprecated) Limits max size of ext type. - (default: max_buffer_size or 1024*1024) + Deprecated, use *max_buffer_size* instead. + Limits max size of ext type. (default: max_buffer_size or 1024*1024) - example of streaming deserialize from file-like object:: + Example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like, raw=False) + unpacker = Unpacker(file_like, raw=False, max_buffer_size=10*1024*1024) for o in unpacker: process(o) - example of streaming deserialize from socket:: + Example of streaming deserialize from socket:: - unpacker = Unpacker(raw=False) + unpacker = Unpacker(raw=False, max_buffer_size=10*1024*1024) while True: buf = sock.recv(1024**2) if not buf: From 8f513af999d4abd39d632fcc8732225a658268ee Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 25 Jan 2019 21:43:28 +0900 Subject: [PATCH 0989/1172] v0.6.1 --- msgpack/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index 49a4103..926c5e7 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 6, 0) +version = (0, 6, 1) From 381c2eff5f8ee0b8669fd6daf1fd1ecaffe7c931 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 4 Feb 2019 12:08:07 +0900 Subject: [PATCH 0990/1172] Fix changelog. Fixes #343 --- ChangeLog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 2c988db..727ca9a 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -3,7 +3,7 @@ Release Date: 2019-01-25 -This release is for mitigating pain caused by v0.6.1 reduced max input limits +This release is for mitigating pain caused by v0.6.0 reduced max input limits for security reason. * ``unpackb(data)`` configures ``max_*_len`` options from ``len(data)``, From 737f08a885dcff32aa1a417a45936d7f7810ee37 Mon Sep 17 00:00:00 2001 From: Hugues Date: Wed, 27 Mar 2019 06:37:26 -0700 Subject: [PATCH 0991/1172] Update requirements.txt (#346) bytearray.pxd is only available starting with Cython 0.29 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cd54e6d..e08dd4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython==0.27.3 +Cython~=0.29.5 From 05ff11dbcc8181cc781b121e46e76a01258a32af Mon Sep 17 00:00:00 2001 From: Felix Schwarz Date: Sun, 12 May 2019 14:44:32 +0200 Subject: [PATCH 0992/1172] use relative imports (#357) Some applications use msgpack to store persistent data and require a specific msgpack version (e.g. borgbackup). Bundling helps in case there is an (incompatible) version of msgpack in a system-wide install. --- msgpack/__init__.py | 10 +++++----- msgpack/_packer.pyx | 2 +- msgpack/_unpacker.pyx | 4 ++-- msgpack/fallback.py | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 7493c4c..4ad9c1a 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,6 +1,6 @@ # coding: utf-8 -from msgpack._version import version -from msgpack.exceptions import * +from ._version import version +from .exceptions import * from collections import namedtuple @@ -19,12 +19,12 @@ class ExtType(namedtuple('ExtType', 'code data')): import os if os.environ.get('MSGPACK_PUREPYTHON'): - from msgpack.fallback import Packer, unpackb, Unpacker + from .fallback import Packer, unpackb, Unpacker else: try: - from msgpack._cmsgpack import Packer, unpackb, Unpacker + from ._cmsgpack import Packer, unpackb, Unpacker except ImportError: - from msgpack.fallback import Packer, unpackb, Unpacker + from .fallback import Packer, unpackb, Unpacker def pack(o, stream, **kwargs): diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index bfde043..dcee213 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -3,7 +3,7 @@ from cpython cimport * from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact -from msgpack import ExtType +from . import ExtType cdef extern from "Python.h": diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 3c6d59e..3727f50 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -12,14 +12,14 @@ from libc.string cimport * from libc.limits cimport * ctypedef unsigned long long uint64_t -from msgpack.exceptions import ( +from .exceptions import ( BufferFull, OutOfData, ExtraData, FormatError, StackError, ) -from msgpack import ExtType +from . import ExtType cdef extern from "unpack.h": diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 1aa3bdf..3836e83 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -59,7 +59,7 @@ else: newlist_hint = lambda size: [] -from msgpack.exceptions import ( +from .exceptions import ( BufferFull, OutOfData, ExtraData, @@ -67,7 +67,7 @@ from msgpack.exceptions import ( StackError, ) -from msgpack import ExtType +from . import ExtType EX_SKIP = 0 From b98b8cab99d7b2dbfe2b2211974564b7e22e9412 Mon Sep 17 00:00:00 2001 From: Marty B Date: Wed, 18 Sep 2019 18:15:09 +0200 Subject: [PATCH 0993/1172] Avoid calling __Pyx_GetModuleGlobalName for ExtType (#363) --- msgpack/_packer.pyx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index dcee213..e275ef2 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -3,6 +3,8 @@ from cpython cimport * from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact +cdef ExtType + from . import ExtType From 3146ebd330cbd02d0d7b4f82a94472cb395804ef Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 19 Sep 2019 13:20:57 +0900 Subject: [PATCH 0994/1172] Use Py_SIZE() when it is safe (#369) --- msgpack/_packer.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index e275ef2..2f4d120 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -200,7 +200,7 @@ cdef class Packer(object): dval = o ret = msgpack_pack_double(&self.pk, dval) elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): - L = len(o) + L = Py_SIZE(o) if L > ITEM_LIMIT: PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) rawval = o @@ -214,7 +214,7 @@ cdef class Packer(object): raise ValueError("unicode string is too large") else: o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) - L = len(o) + L = Py_SIZE(o) if L > ITEM_LIMIT: raise ValueError("unicode string is too large") ret = msgpack_pack_raw(&self.pk, L) @@ -254,7 +254,7 @@ cdef class Packer(object): ret = msgpack_pack_ext(&self.pk, longval, L) ret = msgpack_pack_raw_body(&self.pk, rawval, L) elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): - L = len(o) + L = Py_SIZE(o) if L > ITEM_LIMIT: raise ValueError("list is too large") ret = msgpack_pack_array(&self.pk, L) From c25e2a0984ec5d092fee38eeb4b74676ada9aef4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 19 Sep 2019 20:14:33 +0900 Subject: [PATCH 0995/1172] update Cython to 0.29.13 (#370) --- .travis.yml | 5 +++-- requirements.txt | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 51917c5..43c5259 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ matrix: - DOCKER_IMAGE=quay.io/pypa/manylinux1_i686 install: - pip install -U pip - - pip install cython + - pip install -r requirements.txt - make cython - docker pull $DOCKER_IMAGE script: @@ -39,7 +39,8 @@ matrix: install: - pip install -U pip - - pip install -U cython pytest + - pip install -U pytest + - pip install -r requirements.txt - make cython - pip install -e . diff --git a/requirements.txt b/requirements.txt index e08dd4f..a2cce25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython~=0.29.5 +Cython~=0.29.13 From fd3f0048633423651772526875611f125dda68f6 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 19 Sep 2019 20:37:19 +0900 Subject: [PATCH 0996/1172] Add Python 3.8 to travis (#371) --- .travis.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 43c5259..c80bb37 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,16 +3,20 @@ language: python cache: pip python: + # Available Python (PyPy) can be listed by: + # + # $ aws s3 ls s3://travis-python-archives/binaries/ubuntu/16.04/x86_64/ - "2.7" - "3.4" - "3.5" - "3.6" - "3.7" - - "nightly" + - "3.8-dev" matrix: include: - - sudo: required + - name: 32bit build + sudo: required language: python services: - docker @@ -25,12 +29,14 @@ matrix: - docker pull $DOCKER_IMAGE script: - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh - - python: "pypy2.7-5.10.0" + - name: "pypy2.7" + python: "pypy2.7-7.1.1" install: - pip install -e . script: - py.test -v test - - python: "pypy3.5" + - name: "pypy3" + python: "pypy3.6-7.1.1" install: - pip install -e . script: From 144f276e885be867c1545226a60c99957dac04e0 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 20 Sep 2019 16:36:37 +0900 Subject: [PATCH 0997/1172] Update ChangeLog --- ChangeLog.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index 727ca9a..a295e6b 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,13 @@ +0.6.2 +===== + +Release Date: 2019-09-20 + +* Support Python 3.8. +* Update Cython to 0.29.13 for support Python 3.8. +* Some small optimizations. + + 0.6.1 ====== From 997b524f06176aaa6bd255a046a8746e99b4f87d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 20 Sep 2019 16:37:08 +0900 Subject: [PATCH 0998/1172] 0.6.2 --- msgpack/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index 926c5e7..1e73a00 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 6, 1) +version = (0, 6, 2) From b458e9a6a2cf69e483fa5994d227382c6a01f3c4 Mon Sep 17 00:00:00 2001 From: Terence Honles Date: Fri, 22 Nov 2019 19:58:55 -0800 Subject: [PATCH 0999/1172] update for Python 3.8 (#374) --- appveyor.yml | 4 ++++ docker/buildwheel.sh | 5 ++++- docker/runtests.sh | 5 ++++- docker/shared.env | 8 ++++++++ setup.py | 1 + tox.ini | 2 +- 6 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 docker/shared.env diff --git a/appveyor.yml b/appveyor.yml index bd0800a..f338e17 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -29,6 +29,10 @@ test_script: - ci\\runtests.bat - set PYTHON="C:\\Python37-x64" - ci\\runtests.bat + - set PYTHON="C:\\Python38" + - ci\\runtests.bat + - set PYTHON="C:\\Python38-x64" + - ci\\runtests.bat after_test: # This step builds your wheels. diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh index d8c74cb..c953127 100644 --- a/docker/buildwheel.sh +++ b/docker/buildwheel.sh @@ -1,10 +1,13 @@ #!/bin/bash +DOCKER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$DOCKER_DIR/shared.env" + set -e -x ARCH=`uname -p` echo "arch=$ARCH" -for V in cp37-cp37m cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do +for V in "${PYTHON_VERSIONS[@]}"; do PYBIN=/opt/python/$V/bin rm -rf build/ # Avoid lib build by narrow Python is used by wide python $PYBIN/python setup.py bdist_wheel -p manylinux1_${ARCH} diff --git a/docker/runtests.sh b/docker/runtests.sh index c6bbf60..fa7e979 100755 --- a/docker/runtests.sh +++ b/docker/runtests.sh @@ -1,7 +1,10 @@ #!/bin/bash +DOCKER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$DOCKER_DIR/shared.env" + set -e -x -for V in cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do +for V in "${PYTHON_VERSIONS[@]}"; do PYBIN=/opt/python/$V/bin $PYBIN/python setup.py install rm -rf build/ # Avoid lib build by narrow Python is used by wide python diff --git a/docker/shared.env b/docker/shared.env new file mode 100644 index 0000000..b5772e3 --- /dev/null +++ b/docker/shared.env @@ -0,0 +1,8 @@ +PYTHON_VERSIONS=( + cp38-cp38 + cp37-cp37m + cp36-cp36m + cp35-cp35m + cp27-cp27m + cp27-cp27mu +) diff --git a/setup.py b/setup.py index eb9403f..8b8d7a0 100755 --- a/setup.py +++ b/setup.py @@ -128,6 +128,7 @@ setup(name=name, 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Intended Audience :: Developers', diff --git a/tox.ini b/tox.ini index 0945a6d..4b059ff 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py35,py36,py37}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86 +envlist = {py27,py35,py36,py37,py38}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86 [variants:pure] setenv= From 891f2d8743857bb75204f96b0469cb2ec90c7f79 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 28 Nov 2019 20:23:34 +0900 Subject: [PATCH 1000/1172] Drop Python 2 support from _cmsgpack (#376) --- .travis.yml | 10 ++++++++-- README.rst | 20 +++++++++++--------- docker/shared.env | 2 -- msgpack/__init__.py | 5 +++-- msgpack/_packer.pyx | 5 +---- msgpack/buff_converter.h | 20 -------------------- msgpack/fallback.py | 5 ++--- msgpack/unpack.h | 4 ---- setup.py | 20 +++++++++++--------- 9 files changed, 36 insertions(+), 55 deletions(-) diff --git a/.travis.yml b/.travis.yml index c80bb37..7b298af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,6 @@ python: # Available Python (PyPy) can be listed by: # # $ aws s3 ls s3://travis-python-archives/binaries/ubuntu/16.04/x86_64/ - - "2.7" - "3.4" - "3.5" - "3.6" @@ -41,7 +40,14 @@ matrix: - pip install -e . script: - pytest -v test - + - name: "Python 2 (fallback)" + python: "2.7" + install: + - pip install -U pip + - pip install -U pytest + - pip install . + script: + - pytest -v test install: - pip install -U pip diff --git a/README.rst b/README.rst index 94a4bb2..82b6c02 100644 --- a/README.rst +++ b/README.rst @@ -76,10 +76,18 @@ Install $ pip install msgpack -PyPy -^^^^ +Pure Python implementation +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The extension module in msgpack (``msgpack._cmsgpack``) does not support +Python 2 and PyPy. + +But msgpack provides a pure Python implementation (``msgpack.fallback``) +for PyPy and Python 2. + +Since the [pip](https://pip.pypa.io/) uses the pure Python implementation, +Python 2 support will not be dropped in foreseeable feature. -msgpack provides a pure Python implementation. PyPy can use this. Windows ^^^^^^^ @@ -88,12 +96,6 @@ When you can't use a binary distribution, you need to install Visual Studio or Windows SDK on Windows. Without extension, using pure Python implementation on CPython runs slowly. -For Python 2.7, `Microsoft Visual C++ Compiler for Python 2.7 `_ -is recommended solution. - -For Python 3.5, `Microsoft Visual Studio 2015 `_ -Community Edition or Express Edition can be used to build extension module. - How to use ---------- diff --git a/docker/shared.env b/docker/shared.env index b5772e3..17abdd8 100644 --- a/docker/shared.env +++ b/docker/shared.env @@ -3,6 +3,4 @@ PYTHON_VERSIONS=( cp37-cp37m cp36-cp36m cp35-cp35m - cp27-cp27m - cp27-cp27mu ) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 4ad9c1a..4112a16 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -2,6 +2,8 @@ from ._version import version from .exceptions import * +import os +import sys from collections import namedtuple @@ -17,8 +19,7 @@ class ExtType(namedtuple('ExtType', 'code data')): return super(ExtType, cls).__new__(cls, code, data) -import os -if os.environ.get('MSGPACK_PUREPYTHON'): +if os.environ.get('MSGPACK_PUREPYTHON') or sys.version_info[0] == 2: from .fallback import Packer, unpackb, Unpacker else: try: diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 2f4d120..e620914 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -130,10 +130,7 @@ cdef class Packer(object): self._bencoding = encoding if encoding is None: - if PY_MAJOR_VERSION < 3: - self.encoding = 'utf-8' - else: - self.encoding = NULL + self.encoding = 'utf-8' else: self.encoding = self._bencoding diff --git a/msgpack/buff_converter.h b/msgpack/buff_converter.h index bc7227a..86b4196 100644 --- a/msgpack/buff_converter.h +++ b/msgpack/buff_converter.h @@ -1,28 +1,8 @@ #include "Python.h" /* cython does not support this preprocessor check => write it in raw C */ -#if PY_MAJOR_VERSION == 2 -static PyObject * -buff_to_buff(char *buff, Py_ssize_t size) -{ - return PyBuffer_FromMemory(buff, size); -} - -#elif (PY_MAJOR_VERSION == 3) && (PY_MINOR_VERSION >= 3) static PyObject * buff_to_buff(char *buff, Py_ssize_t size) { return PyMemoryView_FromMemory(buff, size, PyBUF_READ); } -#else -static PyObject * -buff_to_buff(char *buff, Py_ssize_t size) -{ - Py_buffer pybuf; - if (PyBuffer_FillInfo(&pybuf, NULL, buff, size, 1, PyBUF_FULL_RO) == -1) { - return NULL; - } - - return PyMemoryView_FromBuffer(&pybuf); -} -#endif diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 3836e83..1ed6e77 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -5,13 +5,12 @@ import struct import warnings -if sys.version_info[0] == 2: - PY2 = True +PY2 = sys.version_info[0] == 2 +if PY2: int_types = (int, long) def dict_iteritems(d): return d.iteritems() else: - PY2 = False int_types = int unicode = str xrange = range diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 85dbbed..bbce91c 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -273,11 +273,7 @@ static inline int unpack_callback_ext(unpack_user* u, const char* base, const ch return -1; } // length also includes the typecode, so the actual data is length-1 -#if PY_MAJOR_VERSION == 2 - py = PyObject_CallFunction(u->ext_hook, "(is#)", (int)typecode, pos, (Py_ssize_t)length-1); -#else py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1); -#endif if (!py) return -1; *o = py; diff --git a/setup.py b/setup.py index 8b8d7a0..77b81c6 100755 --- a/setup.py +++ b/setup.py @@ -9,6 +9,11 @@ from setuptools import setup, Extension from distutils.command.build_ext import build_ext + +PYPY = hasattr(sys, "pypy_version_info") +PY2 = sys.version_info[0] == 2 + + # for building transitional package. TRANSITIONAL = False @@ -64,14 +69,11 @@ version_str = '.'.join(str(x) for x in version[:3]) if len(version) > 3 and version[3] != 'final': version_str += version[3] -# take care of extension modules. -if have_cython: - class Sdist(sdist): - def __init__(self, *args, **kwargs): - cythonize('msgpack/_cmsgpack.pyx') - sdist.__init__(self, *args, **kwargs) -else: - Sdist = sdist +# Cython is required for sdist +class Sdist(sdist): + def __init__(self, *args, **kwargs): + cythonize('msgpack/_cmsgpack.pyx') + sdist.__init__(self, *args, **kwargs) libraries = [] if sys.platform == 'win32': @@ -83,7 +85,7 @@ else: macros = [('__LITTLE_ENDIAN__', '1')] ext_modules = [] -if not hasattr(sys, 'pypy_version_info'): +if not PYPY and not PY2: ext_modules.append(Extension('msgpack._cmsgpack', sources=['msgpack/_cmsgpack.cpp'], libraries=libraries, From cc3a8665d6210e933bcfb9120bd0ceb15224f03e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 3 Dec 2019 17:46:28 +0900 Subject: [PATCH 1001/1172] Use Github Actions for Windows (#377) --- .github/workflows/windows.yaml | 70 ++++++++++++++++++++++++++++++++++ ci/runtests.bat | 2 +- ci/runtests.sh | 8 ++++ 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/windows.yaml create mode 100644 ci/runtests.sh diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml new file mode 100644 index 0000000..cecb825 --- /dev/null +++ b/.github/workflows/windows.yaml @@ -0,0 +1,70 @@ +name: Build and test windows wheels +on: + push: + branches: + - master + - test + pull_request: + create: + +jobs: + build: + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v1 + + - name: Cythonize + shell: bash + run: | + pip install -U Cython + make cython + #python setup.py sdist + + - name: Python 3.6 (amd64) + env: + PYTHON: "py -3.6-64" + shell: bash + run: | + ci/runtests.sh + + - name: Python 3.6 (x86) + env: + PYTHON: "py -3.6-32" + shell: bash + run: | + ci/runtests.sh + + - name: Python 3.7 (amd64) + env: + PYTHON: "py -3.7-64" + shell: bash + run: | + ci/runtests.sh + + - name: Python 3.7 (x86) + env: + PYTHON: "py -3.7-32" + shell: bash + run: | + ci/runtests.sh + + - name: Python 3.8 (amd64) + env: + PYTHON: "py -3.8-64" + shell: bash + run: | + ci/runtests.sh + + - name: Python 3.8 (x86) + env: + PYTHON: "py -3.8-32" + shell: bash + run: | + ci/runtests.sh + + - name: Upload Wheels + uses: actions/upload-artifact@v1 + with: + name: win-wheels + path: ./dist diff --git a/ci/runtests.bat b/ci/runtests.bat index 0240467..4ae2f70 100644 --- a/ci/runtests.bat +++ b/ci/runtests.bat @@ -2,7 +2,7 @@ %PYTHON%\python.exe setup.py build_ext -i %PYTHON%\python.exe setup.py install %PYTHON%\python.exe -c "import sys; print(hex(sys.maxsize))" -%PYTHON%\python.exe -c "from msgpack import _packer, _unpacker" +%PYTHON%\python.exe -c "from msgpack import _cmsgpack" %PYTHON%\python.exe setup.py bdist_wheel %PYTHON%\python.exe -m pytest -v test SET EL=%ERRORLEVEL% diff --git a/ci/runtests.sh b/ci/runtests.sh new file mode 100644 index 0000000..5d87f69 --- /dev/null +++ b/ci/runtests.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -ex +${PYTHON} -VV +${PYTHON} -m pip install setuptools wheel pytest +${PYTHON} setup.py build_ext -if +${PYTHON} -c "from msgpack import _cmsgpack" +${PYTHON} setup.py bdist_wheel +${PYTHON} -m pytest -v test From e1ed0044bf31dc0d6ef951f6298de4f420170968 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 3 Dec 2019 18:54:01 +0900 Subject: [PATCH 1002/1172] Remove encoding/unicode_errors options from Packer (#378) --- msgpack/_packer.pyx | 46 +++++++-------------------------------------- msgpack/fallback.py | 26 ++----------------------- test/test_pack.py | 30 ----------------------------- 3 files changed, 9 insertions(+), 93 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index e620914..2e698e1 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -89,19 +89,9 @@ cdef class Packer(object): Additionally tuples will not be serialized as lists. This is useful when trying to implement accurate serialization for python types. - - :param str unicode_errors: - Error handler for encoding unicode. (default: 'strict') - - :param str encoding: - (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8') """ cdef msgpack_packer pk cdef object _default - cdef object _bencoding - cdef object _berrors - cdef const char *encoding - cdef const char *unicode_errors cdef bint strict_types cdef bool use_float cdef bint autoreset @@ -114,11 +104,11 @@ cdef class Packer(object): self.pk.buf_size = buf_size self.pk.length = 0 - def __init__(self, default=None, encoding=None, unicode_errors=None, - bint use_single_float=False, bint autoreset=True, bint use_bin_type=False, + def __init__(self, default=None, + bint use_single_float=False, + bint autoreset=True, + bint use_bin_type=False, bint strict_types=False): - if encoding is not None: - PyErr_WarnEx(DeprecationWarning, "encoding is deprecated.", 1) self.use_float = use_single_float self.strict_types = strict_types self.autoreset = autoreset @@ -128,18 +118,6 @@ cdef class Packer(object): raise TypeError("default must be a callable.") self._default = default - self._bencoding = encoding - if encoding is None: - self.encoding = 'utf-8' - else: - self.encoding = self._bencoding - - self._berrors = unicode_errors - if unicode_errors is None: - self.unicode_errors = NULL - else: - self.unicode_errors = self._berrors - def __dealloc__(self): PyMem_Free(self.pk.buf) self.pk.buf = NULL @@ -205,19 +183,9 @@ cdef class Packer(object): if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, L) elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): - if self.encoding == NULL and self.unicode_errors == NULL: - ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); - if ret == -2: - raise ValueError("unicode string is too large") - else: - o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors) - L = Py_SIZE(o) - if L > ITEM_LIMIT: - raise ValueError("unicode string is too large") - ret = msgpack_pack_raw(&self.pk, L) - if ret == 0: - rawval = o - ret = msgpack_pack_raw_body(&self.pk, rawval, L) + ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); + if ret == -2: + raise ValueError("unicode string is too large") elif PyDict_CheckExact(o): d = o L = len(d) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 1ed6e77..5dab906 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -752,32 +752,14 @@ class Packer(object): Additionally tuples will not be serialized as lists. This is useful when trying to implement accurate serialization for python types. - - :param str encoding: - (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8') - - :param str unicode_errors: - Error handler for encoding unicode. (default: 'strict') """ - def __init__(self, default=None, encoding=None, unicode_errors=None, + def __init__(self, default=None, use_single_float=False, autoreset=True, use_bin_type=False, strict_types=False): - if encoding is None: - encoding = 'utf_8' - else: - warnings.warn( - "encoding is deprecated, Use raw=False instead.", - DeprecationWarning, stacklevel=2) - - if unicode_errors is None: - unicode_errors = 'strict' - self._strict_types = strict_types self._use_float = use_single_float self._autoreset = autoreset self._use_bin_type = use_bin_type - self._encoding = encoding - self._unicode_errors = unicode_errors self._buffer = StringIO() if default is not None: if not callable(default): @@ -834,11 +816,7 @@ class Packer(object): self._pack_bin_header(n) return self._buffer.write(obj) if check(obj, unicode): - if self._encoding is None: - raise TypeError( - "Can't encode unicode string: " - "no encoding is specified") - obj = obj.encode(self._encoding, self._unicode_errors) + obj = obj.encode("utf-8") n = len(obj) if n >= 2**32: raise ValueError("String is too large") diff --git a/test/test_pack.py b/test/test_pack.py index 3658a97..194b2c9 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -40,21 +40,6 @@ def testPackUnicode(): re = Unpacker(BytesIO(data), raw=False, use_list=1).unpack() assert re == td -def testPackUTF32(): # deprecated - try: - test_data = [ - "", - "abcd", - ["defgh"], - "РуÑÑкий текÑÑ‚", - ] - for td in test_data: - with pytest.deprecated_call(): - re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32') - assert re == td - except LookupError as e: - xfail(e) - def testPackBytes(): test_data = [ b"", b"abcd", (b"defgh",), @@ -69,26 +54,11 @@ def testPackByteArrays(): for td in test_data: check(td) -def testIgnoreUnicodeErrors(): # deprecated - with pytest.deprecated_call(): - re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1) - assert re == "abcdef" - def testStrictUnicodeUnpack(): packed = packb(b'abc\xeddef') with pytest.raises(UnicodeDecodeError): unpackb(packed, raw=False, use_list=1) -def testStrictUnicodePack(): # deprecated - with raises(UnicodeEncodeError): - with pytest.deprecated_call(): - packb("abc\xeddef", encoding='ascii', unicode_errors='strict') - -def testIgnoreErrorsPack(): # deprecated - with pytest.deprecated_call(): - re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw=False, use_list=1) - assert re == "abcdef" - def testDecodeBinary(): re = unpackb(packb(b"abc"), encoding=None, use_list=1) assert re == b"abc" From a0480c760256b4afc18beaebd5e3c79de1d4ce56 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 3 Dec 2019 18:54:18 +0900 Subject: [PATCH 1003/1172] Update ChangeLog --- ChangeLog.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index a295e6b..1352af8 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,13 @@ +1.0.0 +===== + +Release Date: TBD + +* Remove Python 2 support from the ``msgpack/_cmsgpack``. + ``msgpack/fallback`` still supports Python 2. +* Remove encoding and unicode_errors options from the Packer. + + 0.6.2 ===== From 83ebb63c447a99c81d043eb6808bbfb50697a751 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 3 Dec 2019 20:53:11 +0900 Subject: [PATCH 1004/1172] Ressurect unicode_errors of the Packer. (#379) --- ChangeLog.rst | 2 +- msgpack/_packer.pyx | 34 +++++++++++++++++++++++++++------- msgpack/fallback.py | 11 ++++++++--- test/test_pack.py | 16 ++++++++++++++-- 4 files changed, 50 insertions(+), 13 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 1352af8..1d784af 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -5,7 +5,7 @@ Release Date: TBD * Remove Python 2 support from the ``msgpack/_cmsgpack``. ``msgpack/fallback`` still supports Python 2. -* Remove encoding and unicode_errors options from the Packer. +* Remove ``encoding`` option from the Packer. 0.6.2 diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 2e698e1..8b1a392 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -89,9 +89,15 @@ cdef class Packer(object): Additionally tuples will not be serialized as lists. This is useful when trying to implement accurate serialization for python types. + + :param str unicode_errors: + The error handler for encoding unicode. (default: 'strict') + DO NOT USE THIS!! This option is kept for very specific usage. """ cdef msgpack_packer pk cdef object _default + cdef object _berrors + cdef const char *unicode_errors cdef bint strict_types cdef bool use_float cdef bint autoreset @@ -104,10 +110,8 @@ cdef class Packer(object): self.pk.buf_size = buf_size self.pk.length = 0 - def __init__(self, default=None, - bint use_single_float=False, - bint autoreset=True, - bint use_bin_type=False, + def __init__(self, *, default=None, unicode_errors=None, + bint use_single_float=False, bint autoreset=True, bint use_bin_type=False, bint strict_types=False): self.use_float = use_single_float self.strict_types = strict_types @@ -118,6 +122,12 @@ cdef class Packer(object): raise TypeError("default must be a callable.") self._default = default + self._berrors = unicode_errors + if unicode_errors is None: + self.unicode_errors = NULL + else: + self.unicode_errors = self._berrors + def __dealloc__(self): PyMem_Free(self.pk.buf) self.pk.buf = NULL @@ -183,9 +193,19 @@ cdef class Packer(object): if ret == 0: ret = msgpack_pack_raw_body(&self.pk, rawval, L) elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): - ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); - if ret == -2: - raise ValueError("unicode string is too large") + if self.unicode_errors == NULL: + ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); + if ret == -2: + raise ValueError("unicode string is too large") + else: + o = PyUnicode_AsEncodedString(o, NULL, self.unicode_errors) + L = Py_SIZE(o) + if L > ITEM_LIMIT: + raise ValueError("unicode string is too large") + ret = msgpack_pack_raw(&self.pk, L) + if ret == 0: + rawval = o + ret = msgpack_pack_raw_body(&self.pk, rawval, L) elif PyDict_CheckExact(o): d = o L = len(d) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 5dab906..0c0c101 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -667,7 +667,7 @@ class Unpacker(object): elif self._raw: obj = bytes(obj) else: - obj = obj.decode('utf_8') + obj = obj.decode('utf_8', self._unicode_errors) return obj if typ == TYPE_EXT: return self._ext_hook(n, bytes(obj)) @@ -752,14 +752,19 @@ class Packer(object): Additionally tuples will not be serialized as lists. This is useful when trying to implement accurate serialization for python types. + + :param str unicode_errors: + The error handler for encoding unicode. (default: 'strict') + DO NOT USE THIS!! This option is kept for very specific usage. """ - def __init__(self, default=None, + def __init__(self, default=None, unicode_errors=None, use_single_float=False, autoreset=True, use_bin_type=False, strict_types=False): self._strict_types = strict_types self._use_float = use_single_float self._autoreset = autoreset self._use_bin_type = use_bin_type + self._unicode_errors = unicode_errors or "strict" self._buffer = StringIO() if default is not None: if not callable(default): @@ -816,7 +821,7 @@ class Packer(object): self._pack_bin_header(n) return self._buffer.write(obj) if check(obj, unicode): - obj = obj.encode("utf-8") + obj = obj.encode("utf-8", self._unicode_errors) n = len(obj) if n >= 2**32: raise ValueError("String is too large") diff --git a/test/test_pack.py b/test/test_pack.py index 194b2c9..b6752e5 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera from collections import OrderedDict from io import BytesIO import struct +import sys import pytest from pytest import raises, xfail @@ -54,13 +55,24 @@ def testPackByteArrays(): for td in test_data: check(td) +@pytest.mark.skipif(sys.version_info < (3,0), reason="Python 2 passes invalid surrogates") +def testIgnoreUnicodeErrors(): + re = unpackb(packb(b'abc\xeddef', use_bin_type=False), + raw=False, unicode_errors='ignore') + assert re == "abcdef" + def testStrictUnicodeUnpack(): - packed = packb(b'abc\xeddef') + packed = packb(b'abc\xeddef', use_bin_type=False) with pytest.raises(UnicodeDecodeError): unpackb(packed, raw=False, use_list=1) +@pytest.mark.skipif(sys.version_info < (3,0), reason="Python 2 passes invalid surrogates") +def testIgnoreErrorsPack(): + re = unpackb(packb(u"abc\uDC80\uDCFFdef", use_bin_type=True, unicode_errors='ignore'), raw=False, use_list=1) + assert re == "abcdef" + def testDecodeBinary(): - re = unpackb(packb(b"abc"), encoding=None, use_list=1) + re = unpackb(packb(b"abc"), use_list=1) assert re == b"abc" def testPackFloat(): From e419cd8e2db6b8226bd681b52b6acfe70d8e6a86 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 3 Dec 2019 21:13:05 +0900 Subject: [PATCH 1005/1172] Remove encoding option from Unpacker. (#380) --- ChangeLog.rst | 2 +- msgpack/_unpacker.pyx | 36 ++++++++---------------------------- msgpack/fallback.py | 23 +++++------------------ msgpack/unpack.h | 5 +---- 4 files changed, 15 insertions(+), 51 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 1d784af..d44b36a 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -5,7 +5,7 @@ Release Date: TBD * Remove Python 2 support from the ``msgpack/_cmsgpack``. ``msgpack/fallback`` still supports Python 2. -* Remove ``encoding`` option from the Packer. +* Remove ``encoding`` option from the Packer and Unpacker. 0.6.2 diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 3727f50..b258686 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -31,7 +31,6 @@ cdef extern from "unpack.h": PyObject* object_hook PyObject* list_hook PyObject* ext_hook - char *encoding char *unicode_errors Py_ssize_t max_str_len Py_ssize_t max_bin_len @@ -58,7 +57,7 @@ cdef inline init_ctx(unpack_context *ctx, object object_hook, object object_pairs_hook, object list_hook, object ext_hook, bint use_list, bint raw, bint strict_map_key, - const char* encoding, const char* unicode_errors, + const char* unicode_errors, Py_ssize_t max_str_len, Py_ssize_t max_bin_len, Py_ssize_t max_array_len, Py_ssize_t max_map_len, Py_ssize_t max_ext_len): @@ -99,7 +98,6 @@ cdef inline init_ctx(unpack_context *ctx, raise TypeError("ext_hook must be a callable.") ctx.user.ext_hook = ext_hook - ctx.user.encoding = encoding ctx.user.unicode_errors = unicode_errors def default_read_extended_type(typecode, data): @@ -141,9 +139,9 @@ cdef inline int get_data_from_buffer(object obj, 1) return 1 -def unpackb(object packed, object object_hook=None, object list_hook=None, +def unpackb(object packed, *, object object_hook=None, object list_hook=None, bint use_list=True, bint raw=True, bint strict_map_key=False, - encoding=None, unicode_errors=None, + unicode_errors=None, object_pairs_hook=None, ext_hook=ExtType, Py_ssize_t max_str_len=-1, Py_ssize_t max_bin_len=-1, @@ -170,14 +168,9 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, cdef Py_buffer view cdef char* buf = NULL cdef Py_ssize_t buf_len - cdef const char* cenc = NULL cdef const char* cerr = NULL cdef int new_protocol = 0 - if encoding is not None: - PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) - cenc = encoding - if unicode_errors is not None: cerr = unicode_errors @@ -196,7 +189,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, try: init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook, - use_list, raw, strict_map_key, cenc, cerr, + use_list, raw, strict_map_key, cerr, max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) ret = unpack_construct(&ctx, buf, buf_len, &off) finally: @@ -250,8 +243,6 @@ cdef class Unpacker(object): near future. So you must specify it explicitly for keeping backward compatibility. - *encoding* option which is deprecated overrides this option. - :param bool strict_map_key: If true, only str or bytes are accepted for map (dict) keys. It's False by default for backward-compatibility. @@ -290,11 +281,6 @@ cdef class Unpacker(object): Deprecated, use *max_buffer_size* instead. Limits max size of ext type. (default: max_buffer_size or 1024*1024) - :param str encoding: - Deprecated, use ``raw=False`` instead. - Encoding used for decoding msgpack raw. - If it is None (default), msgpack raw is deserialized to Python bytes. - :param str unicode_errors: Error handler used for decoding str type. (default: `'strict'`) @@ -330,7 +316,7 @@ cdef class Unpacker(object): cdef Py_ssize_t read_size # To maintain refcnt. cdef object object_hook, object_pairs_hook, list_hook, ext_hook - cdef object encoding, unicode_errors + cdef object unicode_errors cdef Py_ssize_t max_buffer_size cdef uint64_t stream_offset @@ -341,17 +327,16 @@ cdef class Unpacker(object): PyMem_Free(self.buf) self.buf = NULL - def __init__(self, file_like=None, Py_ssize_t read_size=0, + def __init__(self, file_like=None, *, Py_ssize_t read_size=0, bint use_list=True, bint raw=True, bint strict_map_key=False, object object_hook=None, object object_pairs_hook=None, object list_hook=None, - encoding=None, unicode_errors=None, Py_ssize_t max_buffer_size=0, + unicode_errors=None, Py_ssize_t max_buffer_size=0, object ext_hook=ExtType, Py_ssize_t max_str_len=-1, Py_ssize_t max_bin_len=-1, Py_ssize_t max_array_len=-1, Py_ssize_t max_map_len=-1, Py_ssize_t max_ext_len=-1): - cdef const char *cenc=NULL, cdef const char *cerr=NULL self.object_hook = object_hook @@ -392,17 +377,12 @@ cdef class Unpacker(object): self.buf_tail = 0 self.stream_offset = 0 - if encoding is not None: - PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1) - self.encoding = encoding - cenc = encoding - if unicode_errors is not None: self.unicode_errors = unicode_errors cerr = unicode_errors init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, - ext_hook, use_list, raw, strict_map_key, cenc, cerr, + ext_hook, use_list, raw, strict_map_key, cerr, max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 0c0c101..9e31213 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -176,8 +176,6 @@ class Unpacker(object): near future. So you must specify it explicitly for keeping backward compatibility. - *encoding* option which is deprecated overrides this option. - :param bool strict_map_key: If true, only str or bytes are accepted for map (dict) keys. It's False by default for backward-compatibility. @@ -193,13 +191,10 @@ class Unpacker(object): Unpacker calls it with a list of key-value pairs after unpacking msgpack map. (See also simplejson) - :param str encoding: - Encoding used for decoding msgpack raw. - If it is None (default), msgpack raw is deserialized to Python bytes. - :param str unicode_errors: - (deprecated) Used for decoding msgpack raw with *encoding*. - (default: `'strict'`) + The error handler for decoding unicode. (default: 'strict') + This option should be used only when you have msgpack data which + contains invalid UTF-8 string. :param int max_buffer_size: Limits size of data waiting unpacked. 0 means system's INT_MAX (default). @@ -252,18 +247,13 @@ class Unpacker(object): def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, strict_map_key=False, object_hook=None, object_pairs_hook=None, list_hook=None, - encoding=None, unicode_errors=None, max_buffer_size=0, + unicode_errors=None, max_buffer_size=0, ext_hook=ExtType, max_str_len=-1, max_bin_len=-1, max_array_len=-1, max_map_len=-1, max_ext_len=-1): - if encoding is not None: - warnings.warn( - "encoding is deprecated, Use raw=False instead.", - DeprecationWarning, stacklevel=2) - if unicode_errors is None: unicode_errors = 'strict' @@ -306,7 +296,6 @@ class Unpacker(object): self._read_size = read_size or min(self._max_buffer_size, 16*1024) self._raw = bool(raw) self._strict_map_key = bool(strict_map_key) - self._encoding = encoding self._unicode_errors = unicode_errors self._use_list = use_list self._list_hook = list_hook @@ -662,9 +651,7 @@ class Unpacker(object): if execute == EX_SKIP: return if typ == TYPE_RAW: - if self._encoding is not None: - obj = obj.decode(self._encoding, self._unicode_errors) - elif self._raw: + if self._raw: obj = bytes(obj) else: obj = obj.decode('utf_8', self._unicode_errors) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index bbce91c..539a991 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -27,7 +27,6 @@ typedef struct unpack_user { PyObject *object_hook; PyObject *list_hook; PyObject *ext_hook; - const char *encoding; const char *unicode_errors; Py_ssize_t max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len; } unpack_user; @@ -232,9 +231,7 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* PyObject *py; - if (u->encoding) { - py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors); - } else if (u->raw) { + if (u->raw) { py = PyBytes_FromStringAndSize(p, l); } else { py = PyUnicode_DecodeUTF8(p, l, u->unicode_errors); From 2c6668941f72e3bcb797d096437683eca4e3caf5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 3 Dec 2019 21:18:17 +0900 Subject: [PATCH 1006/1172] Intern map keys (#381) Fixes #372. --- msgpack/fallback.py | 2 ++ msgpack/unpack.h | 3 +++ 2 files changed, 5 insertions(+) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 9e31213..9a48b71 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -644,6 +644,8 @@ class Unpacker(object): key = self._unpack(EX_CONSTRUCT) if self._strict_map_key and type(key) not in (unicode, bytes): raise ValueError("%s is not allowed for map key" % str(type(key))) + if not PY2 and type(key) is str: + key = sys.intern(key) ret[key] = self._unpack(EX_CONSTRUCT) if self._object_hook is not None: ret = self._object_hook(ret) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 539a991..ead5095 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -192,6 +192,9 @@ static inline int unpack_callback_map_item(unpack_user* u, unsigned int current, PyErr_Format(PyExc_ValueError, "%.100s is not allowed for map key", Py_TYPE(k)->tp_name); return -1; } + if (PyUnicode_CheckExact(k)) { + PyUnicode_InternInPlace(&k); + } if (u->has_pairs_hook) { msgpack_unpack_object item = PyTuple_Pack(2, k, v); if (!item) From 641406902efaa8f22f4a7973d04c921a2a35a6be Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 5 Dec 2019 18:29:15 +0900 Subject: [PATCH 1007/1172] Add Timestamp support (#382) --- docs/api.rst | 4 ++ msgpack/__init__.py | 14 +---- msgpack/_packer.pyx | 9 ++- msgpack/_unpacker.pyx | 5 +- msgpack/ext.py | 136 ++++++++++++++++++++++++++++++++++++++++ msgpack/fallback.py | 12 ++-- msgpack/pack_template.h | 33 ++++++++++ msgpack/unpack.h | 44 ++++++++++++- test/test_timestamp.py | 46 ++++++++++++++ 9 files changed, 283 insertions(+), 20 deletions(-) create mode 100644 msgpack/ext.py create mode 100644 test/test_timestamp.py diff --git a/docs/api.rst b/docs/api.rst index 6336793..93827e1 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -27,6 +27,10 @@ API reference .. autoclass:: ExtType +.. autoclass:: Timestamp + :members: + :special-members: __init__ + exceptions ---------- diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 4112a16..ff66f46 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,22 +1,10 @@ # coding: utf-8 from ._version import version from .exceptions import * +from .ext import ExtType, Timestamp import os import sys -from collections import namedtuple - - -class ExtType(namedtuple('ExtType', 'code data')): - """ExtType represents ext type in msgpack.""" - def __new__(cls, code, data): - if not isinstance(code, int): - raise TypeError("code must be int") - if not isinstance(data, bytes): - raise TypeError("data must be bytes") - if not 0 <= code <= 127: - raise ValueError("code must be 0~127") - return super(ExtType, cls).__new__(cls, code, data) if os.environ.get('MSGPACK_PUREPYTHON') or sys.version_info[0] == 2: diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 8b1a392..f3bde3f 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -4,8 +4,9 @@ from cpython cimport * from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact cdef ExtType +cdef Timestamp -from . import ExtType +from .ext import ExtType, Timestamp cdef extern from "Python.h": @@ -36,6 +37,7 @@ cdef extern from "pack.h": int msgpack_pack_bin(msgpack_packer* pk, size_t l) int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l) + int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds); int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit) cdef extern from "buff_converter.h": @@ -135,6 +137,7 @@ cdef class Packer(object): cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: cdef long long llval cdef unsigned long long ullval + cdef unsigned long ulval cdef long longval cdef float fval cdef double dval @@ -238,6 +241,10 @@ cdef class Packer(object): raise ValueError("EXT data is too large") ret = msgpack_pack_ext(&self.pk, longval, L) ret = msgpack_pack_raw_body(&self.pk, rawval, L) + elif type(o) is Timestamp: + llval = o.seconds + ulval = o.nanoseconds + ret = msgpack_pack_timestamp(&self.pk, llval, ulval) elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): L = Py_SIZE(o) if L > ITEM_LIMIT: diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index b258686..6dedd30 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -19,7 +19,7 @@ from .exceptions import ( FormatError, StackError, ) -from . import ExtType +from .ext import ExtType, Timestamp cdef extern from "unpack.h": @@ -31,6 +31,7 @@ cdef extern from "unpack.h": PyObject* object_hook PyObject* list_hook PyObject* ext_hook + PyObject* timestamp_t char *unicode_errors Py_ssize_t max_str_len Py_ssize_t max_bin_len @@ -98,6 +99,8 @@ cdef inline init_ctx(unpack_context *ctx, raise TypeError("ext_hook must be a callable.") ctx.user.ext_hook = ext_hook + # Add Timestamp type to the user object so it may be used in unpack.h + ctx.user.timestamp_t = Timestamp ctx.user.unicode_errors = unicode_errors def default_read_extended_type(typecode, data): diff --git a/msgpack/ext.py b/msgpack/ext.py new file mode 100644 index 0000000..1a0f8fe --- /dev/null +++ b/msgpack/ext.py @@ -0,0 +1,136 @@ +# coding: utf-8 +from collections import namedtuple +import sys +import struct + + +PY2 = sys.version_info[0] == 2 +if not PY2: + long = int + + +class ExtType(namedtuple('ExtType', 'code data')): + """ExtType represents ext type in msgpack.""" + def __new__(cls, code, data): + if not isinstance(code, int): + raise TypeError("code must be int") + if not isinstance(data, bytes): + raise TypeError("data must be bytes") + if code == -1: + return Timestamp.from_bytes(data) + if not 0 <= code <= 127: + raise ValueError("code must be 0~127") + return super(ExtType, cls).__new__(cls, code, data) + + +class Timestamp(object): + """Timestamp represents the Timestamp extension type in msgpack. + + When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. When using pure-Python + msgpack, :func:`to_bytes` and :func:`from_bytes` are used to pack and unpack `Timestamp`. + """ + __slots__ = ["seconds", "nanoseconds"] + + def __init__(self, seconds, nanoseconds=0): + """Initialize a Timestamp object. + + :param seconds: Number of seconds since the UNIX epoch (00:00:00 UTC Jan 1 1970, minus leap seconds). May be + negative. If :code:`seconds` includes a fractional part, :code:`nanoseconds` must be 0. + :type seconds: int or float + + :param nanoseconds: Number of nanoseconds to add to `seconds` to get fractional time. Maximum is 999_999_999. + Default is 0. + :type nanoseconds: int + + Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns. + """ + if not isinstance(seconds, (int, long, float)): + raise TypeError("seconds must be numeric") + if not isinstance(nanoseconds, (int, long)): + raise TypeError("nanoseconds must be an integer") + if nanoseconds: + if nanoseconds < 0 or nanoseconds % 1 != 0 or nanoseconds > (1e9 - 1): + raise ValueError("nanoseconds must be a non-negative integer less than 999999999.") + if not isinstance(seconds, (int, long)): + raise ValueError("seconds must be an integer if also providing nanoseconds.") + self.nanoseconds = nanoseconds + else: + # round helps with floating point issues + self.nanoseconds = int(round(seconds % 1 * 1e9, 0)) + self.seconds = int(seconds // 1) + + def __repr__(self): + """String representation of Timestamp.""" + return "Timestamp(seconds={0}, nanoseconds={1})".format(self.seconds, self.nanoseconds) + + def __eq__(self, other): + """Check for equality with another Timestamp object""" + if type(other) is self.__class__: + return self.seconds == other.seconds and self.nanoseconds == other.nanoseconds + return False + + def __ne__(self, other): + """not-equals method (see :func:`__eq__()`)""" + return not self.__eq__(other) + + @staticmethod + def from_bytes(b): + """Unpack bytes into a `Timestamp` object. + + Used for pure-Python msgpack unpacking. + + :param b: Payload from msgpack ext message with code -1 + :type b: bytes + + :returns: Timestamp object unpacked from msgpack ext payload + :rtype: Timestamp + """ + if len(b) == 4: + seconds = struct.unpack("!L", b)[0] + nanoseconds = 0 + elif len(b) == 8: + data64 = struct.unpack("!Q", b)[0] + seconds = data64 & 0x00000003ffffffff + nanoseconds = data64 >> 34 + elif len(b) == 12: + nanoseconds, seconds = struct.unpack("!Iq", b) + else: + raise ValueError("Timestamp type can only be created from 32, 64, or 96-bit byte objects") + return Timestamp(seconds, nanoseconds) + + def to_bytes(self): + """Pack this Timestamp object into bytes. + + Used for pure-Python msgpack packing. + + :returns data: Payload for EXT message with code -1 (timestamp type) + :rtype: bytes + """ + if (self.seconds >> 34) == 0: # seconds is non-negative and fits in 34 bits + data64 = self.nanoseconds << 34 | self.seconds + if data64 & 0xffffffff00000000 == 0: + # nanoseconds is zero and seconds < 2**32, so timestamp 32 + data = struct.pack("!L", data64) + else: + # timestamp 64 + data = struct.pack("!Q", data64) + else: + # timestamp 96 + data = struct.pack("!Iq", self.nanoseconds, self.seconds) + return data + + def to_float_s(self): + """Get the timestamp as a floating-point value. + + :returns: posix timestamp + :rtype: float + """ + return self.seconds + self.nanoseconds/1e9 + + def to_unix_ns(self): + """Get the timestamp as a unixtime in nanoseconds. + + :returns: posix timestamp in nanoseconds + :rtype: int + """ + return int(self.seconds * 1e9 + self.nanoseconds) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 9a48b71..55e66f5 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -66,7 +66,7 @@ from .exceptions import ( StackError, ) -from . import ExtType +from .ext import ExtType, Timestamp EX_SKIP = 0 @@ -826,9 +826,13 @@ class Packer(object): if self._use_float: return self._buffer.write(struct.pack(">Bf", 0xca, obj)) return self._buffer.write(struct.pack(">Bd", 0xcb, obj)) - if check(obj, ExtType): - code = obj.code - data = obj.data + if check(obj, (ExtType, Timestamp)): + if check(obj, Timestamp): + code = -1 + data = obj.to_bytes() + else: + code = obj.code + data = obj.data assert isinstance(code, int) assert isinstance(data, bytes) L = len(data) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 69982f4..0e940b8 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -759,6 +759,39 @@ static inline int msgpack_pack_ext(msgpack_packer* x, char typecode, size_t l) } +/* + * Pack Timestamp extension type. Follows msgpack-c pack_template.h. + */ +static inline int msgpack_pack_timestamp(msgpack_packer* x, int64_t seconds, uint32_t nanoseconds) +{ + if ((seconds >> 34) == 0) { + /* seconds is unsigned and fits in 34 bits */ + uint64_t data64 = ((uint64_t)nanoseconds << 34) | (uint64_t)seconds; + if ((data64 & 0xffffffff00000000L) == 0) { + /* no nanoseconds and seconds is 32bits or smaller. timestamp32. */ + unsigned char buf[4]; + uint32_t data32 = (uint32_t)data64; + msgpack_pack_ext(x, -1, 4); + _msgpack_store32(buf, data32); + msgpack_pack_raw_body(x, buf, 4); + } else { + /* timestamp64 */ + unsigned char buf[8]; + msgpack_pack_ext(x, -1, 8); + _msgpack_store64(buf, data64); + msgpack_pack_raw_body(x, buf, 8); + + } + } else { + /* seconds is signed or >34bits */ + unsigned char buf[12]; + _msgpack_store32(&buf[0], nanoseconds); + _msgpack_store64(&buf[4], seconds); + msgpack_pack_ext(x, -1, 12); + msgpack_pack_raw_body(x, buf, 12); + } + return 0; +} #undef msgpack_pack_append_buffer diff --git a/msgpack/unpack.h b/msgpack/unpack.h index ead5095..4380ec5 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -27,6 +27,7 @@ typedef struct unpack_user { PyObject *object_hook; PyObject *list_hook; PyObject *ext_hook; + PyObject *timestamp_t; const char *unicode_errors; Py_ssize_t max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len; } unpack_user; @@ -259,6 +260,38 @@ static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* return 0; } +typedef struct msgpack_timestamp { + int64_t tv_sec; + uint32_t tv_nsec; +} msgpack_timestamp; + +/* + * Unpack ext buffer to a timestamp. Pulled from msgpack-c timestamp.h. + */ +static inline int unpack_timestamp(const char* buf, unsigned int buflen, msgpack_timestamp* ts) { + switch (buflen) { + case 4: + ts->tv_nsec = 0; + { + uint32_t v = _msgpack_load32(uint32_t, buf); + ts->tv_sec = (int64_t)v; + } + return 0; + case 8: { + uint64_t value =_msgpack_load64(uint64_t, buf); + ts->tv_nsec = (uint32_t)(value >> 34); + ts->tv_sec = value & 0x00000003ffffffffLL; + return 0; + } + case 12: + ts->tv_nsec = _msgpack_load32(uint32_t, buf); + ts->tv_sec = _msgpack_load64(int64_t, buf + 4); + return 0; + default: + return -1; + } +} + static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos, unsigned int length, msgpack_unpack_object* o) { @@ -273,7 +306,16 @@ static inline int unpack_callback_ext(unpack_user* u, const char* base, const ch return -1; } // length also includes the typecode, so the actual data is length-1 - py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1); + if (typecode == -1) { + msgpack_timestamp ts; + if (unpack_timestamp(pos, length-1, &ts) == 0) { + py = PyObject_CallFunction(u->timestamp_t, "(Lk)", ts.tv_sec, ts.tv_nsec); + } else { + py = NULL; + } + } else { + py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1); + } if (!py) return -1; *o = py; diff --git a/test/test_timestamp.py b/test/test_timestamp.py new file mode 100644 index 0000000..55c2f6d --- /dev/null +++ b/test/test_timestamp.py @@ -0,0 +1,46 @@ +import msgpack +from msgpack import Timestamp + + +def test_timestamp(): + # timestamp32 + ts = Timestamp(2**32 - 1) + assert ts.to_bytes() == b"\xff\xff\xff\xff" + packed = msgpack.packb(ts) + assert packed == b"\xd6\xff" + ts.to_bytes() + unpacked = msgpack.unpackb(packed) + assert ts == unpacked + assert ts.seconds == 2**32 - 1 and ts.nanoseconds == 0 + + # timestamp64 + ts = Timestamp(2**34 - 1, 999999999) + assert ts.to_bytes() == b"\xee\x6b\x27\xff\xff\xff\xff\xff" + packed = msgpack.packb(ts) + assert packed == b"\xd7\xff" + ts.to_bytes() + unpacked = msgpack.unpackb(packed) + assert ts == unpacked + assert ts.seconds == 2**34 - 1 and ts.nanoseconds == 999999999 + + # timestamp96 + ts = Timestamp(2**63 - 1, 999999999) + assert ts.to_bytes() == b"\x3b\x9a\xc9\xff\x7f\xff\xff\xff\xff\xff\xff\xff" + packed = msgpack.packb(ts) + assert packed == b"\xc7\x0c\xff" + ts.to_bytes() + unpacked = msgpack.unpackb(packed) + assert ts == unpacked + assert ts.seconds == 2**63 - 1 and ts.nanoseconds == 999999999 + + # negative fractional + ts = Timestamp(-2.3) #s: -3, ns: 700000000 + assert ts.to_bytes() == b"\x29\xb9\x27\x00\xff\xff\xff\xff\xff\xff\xff\xfd" + packed = msgpack.packb(ts) + assert packed == b"\xc7\x0c\xff" + ts.to_bytes() + unpacked = msgpack.unpackb(packed) + assert ts == unpacked + assert ts.seconds == -3 and ts.nanoseconds == 700000000 + + +def test_timestamp_to(): + t = Timestamp(42, 14) + assert t.to_float_s() == 42.000000014 + assert t.to_unix_ns() == 42000000014 From e557e17cbd4e88622e48547ac52834e9ab95f946 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 5 Dec 2019 18:50:13 +0900 Subject: [PATCH 1008/1172] blacken --- msgpack/__init__.py | 2 +- msgpack/ext.py | 30 +++- msgpack/fallback.py | 393 +++++++++++++++++++++++++------------------- 3 files changed, 245 insertions(+), 180 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index ff66f46..d6705e2 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -7,7 +7,7 @@ import os import sys -if os.environ.get('MSGPACK_PUREPYTHON') or sys.version_info[0] == 2: +if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: from .fallback import Packer, unpackb, Unpacker else: try: diff --git a/msgpack/ext.py b/msgpack/ext.py index 1a0f8fe..c7efff6 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -9,8 +9,9 @@ if not PY2: long = int -class ExtType(namedtuple('ExtType', 'code data')): +class ExtType(namedtuple("ExtType", "code data")): """ExtType represents ext type in msgpack.""" + def __new__(cls, code, data): if not isinstance(code, int): raise TypeError("code must be int") @@ -29,6 +30,7 @@ class Timestamp(object): When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. When using pure-Python msgpack, :func:`to_bytes` and :func:`from_bytes` are used to pack and unpack `Timestamp`. """ + __slots__ = ["seconds", "nanoseconds"] def __init__(self, seconds, nanoseconds=0): @@ -50,9 +52,13 @@ class Timestamp(object): raise TypeError("nanoseconds must be an integer") if nanoseconds: if nanoseconds < 0 or nanoseconds % 1 != 0 or nanoseconds > (1e9 - 1): - raise ValueError("nanoseconds must be a non-negative integer less than 999999999.") + raise ValueError( + "nanoseconds must be a non-negative integer less than 999999999." + ) if not isinstance(seconds, (int, long)): - raise ValueError("seconds must be an integer if also providing nanoseconds.") + raise ValueError( + "seconds must be an integer if also providing nanoseconds." + ) self.nanoseconds = nanoseconds else: # round helps with floating point issues @@ -61,12 +67,16 @@ class Timestamp(object): def __repr__(self): """String representation of Timestamp.""" - return "Timestamp(seconds={0}, nanoseconds={1})".format(self.seconds, self.nanoseconds) + return "Timestamp(seconds={0}, nanoseconds={1})".format( + self.seconds, self.nanoseconds + ) def __eq__(self, other): """Check for equality with another Timestamp object""" if type(other) is self.__class__: - return self.seconds == other.seconds and self.nanoseconds == other.nanoseconds + return ( + self.seconds == other.seconds and self.nanoseconds == other.nanoseconds + ) return False def __ne__(self, other): @@ -90,12 +100,14 @@ class Timestamp(object): nanoseconds = 0 elif len(b) == 8: data64 = struct.unpack("!Q", b)[0] - seconds = data64 & 0x00000003ffffffff + seconds = data64 & 0x00000003FFFFFFFF nanoseconds = data64 >> 34 elif len(b) == 12: nanoseconds, seconds = struct.unpack("!Iq", b) else: - raise ValueError("Timestamp type can only be created from 32, 64, or 96-bit byte objects") + raise ValueError( + "Timestamp type can only be created from 32, 64, or 96-bit byte objects" + ) return Timestamp(seconds, nanoseconds) def to_bytes(self): @@ -108,7 +120,7 @@ class Timestamp(object): """ if (self.seconds >> 34) == 0: # seconds is non-negative and fits in 34 bits data64 = self.nanoseconds << 34 | self.seconds - if data64 & 0xffffffff00000000 == 0: + if data64 & 0xFFFFFFFF00000000 == 0: # nanoseconds is zero and seconds < 2**32, so timestamp 32 data = struct.pack("!L", data64) else: @@ -125,7 +137,7 @@ class Timestamp(object): :returns: posix timestamp :rtype: float """ - return self.seconds + self.nanoseconds/1e9 + return self.seconds + self.nanoseconds / 1e9 def to_unix_ns(self): """Get the timestamp as a unixtime in nanoseconds. diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 55e66f5..577e571 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -8,53 +8,72 @@ import warnings PY2 = sys.version_info[0] == 2 if PY2: int_types = (int, long) + def dict_iteritems(d): return d.iteritems() + + else: int_types = int unicode = str xrange = range + def dict_iteritems(d): return d.items() + if sys.version_info < (3, 5): # Ugly hack... RecursionError = RuntimeError def _is_recursionerror(e): - return len(e.args) == 1 and isinstance(e.args[0], str) and \ - e.args[0].startswith('maximum recursion depth exceeded') + return ( + len(e.args) == 1 + and isinstance(e.args[0], str) + and e.args[0].startswith("maximum recursion depth exceeded") + ) + + else: + def _is_recursionerror(e): return True -if hasattr(sys, 'pypy_version_info'): + +if hasattr(sys, "pypy_version_info"): # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own # StringBuilder is fastest. from __pypy__ import newlist_hint + try: from __pypy__.builders import BytesBuilder as StringBuilder except ImportError: from __pypy__.builders import StringBuilder USING_STRINGBUILDER = True + class StringIO(object): - def __init__(self, s=b''): + def __init__(self, s=b""): if s: self.builder = StringBuilder(len(s)) self.builder.append(s) else: self.builder = StringBuilder() + def write(self, s): if isinstance(s, memoryview): s = s.tobytes() elif isinstance(s, bytearray): s = bytes(s) self.builder.append(s) + def getvalue(self): return self.builder.build() + + else: USING_STRINGBUILDER = False from io import BytesIO as StringIO + newlist_hint = lambda size: [] @@ -69,17 +88,17 @@ from .exceptions import ( from .ext import ExtType, Timestamp -EX_SKIP = 0 -EX_CONSTRUCT = 1 -EX_READ_ARRAY_HEADER = 2 -EX_READ_MAP_HEADER = 3 +EX_SKIP = 0 +EX_CONSTRUCT = 1 +EX_READ_ARRAY_HEADER = 2 +EX_READ_MAP_HEADER = 3 -TYPE_IMMEDIATE = 0 -TYPE_ARRAY = 1 -TYPE_MAP = 2 -TYPE_RAW = 3 -TYPE_BIN = 4 -TYPE_EXT = 5 +TYPE_IMMEDIATE = 0 +TYPE_ARRAY = 1 +TYPE_MAP = 2 +TYPE_RAW = 3 +TYPE_BIN = 4 +TYPE_EXT = 5 DEFAULT_RECURSE_LIMIT = 511 @@ -98,10 +117,13 @@ def _get_data_from_buffer(obj): # try to use legacy buffer protocol if 2.7, otherwise re-raise if PY2: view = memoryview(buffer(obj)) - warnings.warn("using old buffer interface to unpack %s; " - "this leads to unpacking errors if slicing is used and " - "will be removed in a future version" % type(obj), - RuntimeWarning, stacklevel=3) + warnings.warn( + "using old buffer interface to unpack %s; " + "this leads to unpacking errors if slicing is used and " + "will be removed in a future version" % type(obj), + RuntimeWarning, + stacklevel=3, + ) else: raise if view.itemsize != 1: @@ -112,7 +134,9 @@ def _get_data_from_buffer(obj): def unpack(stream, **kwargs): warnings.warn( "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", - DeprecationWarning, stacklevel=2) + DeprecationWarning, + stacklevel=2, + ) data = stream.read() return unpackb(data, **kwargs) @@ -145,9 +169,12 @@ def unpackb(packed, **kwargs): if sys.version_info < (2, 7, 6): + def _unpack_from(f, b, o=0): """Explicit typcast for legacy struct.unpack_from""" return struct.unpack_from(f, bytes(b), o) + + else: _unpack_from = struct.unpack_from @@ -245,17 +272,27 @@ class Unpacker(object): Other exceptions can be raised during unpacking. """ - def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, strict_map_key=False, - object_hook=None, object_pairs_hook=None, list_hook=None, - unicode_errors=None, max_buffer_size=0, - ext_hook=ExtType, - max_str_len=-1, - max_bin_len=-1, - max_array_len=-1, - max_map_len=-1, - max_ext_len=-1): + def __init__( + self, + file_like=None, + read_size=0, + use_list=True, + raw=True, + strict_map_key=False, + object_hook=None, + object_pairs_hook=None, + list_hook=None, + unicode_errors=None, + max_buffer_size=0, + ext_hook=ExtType, + max_str_len=-1, + max_bin_len=-1, + max_array_len=-1, + max_map_len=-1, + max_ext_len=-1, + ): if unicode_errors is None: - unicode_errors = 'strict' + unicode_errors = "strict" if file_like is None: self._feeding = True @@ -280,20 +317,20 @@ class Unpacker(object): self._buf_checkpoint = 0 if max_str_len == -1: - max_str_len = max_buffer_size or 1024*1024 + max_str_len = max_buffer_size or 1024 * 1024 if max_bin_len == -1: - max_bin_len = max_buffer_size or 1024*1024 + max_bin_len = max_buffer_size or 1024 * 1024 if max_array_len == -1: - max_array_len = max_buffer_size or 128*1024 + max_array_len = max_buffer_size or 128 * 1024 if max_map_len == -1: - max_map_len = max_buffer_size//2 or 32*1024 + max_map_len = max_buffer_size // 2 or 32 * 1024 if max_ext_len == -1: - max_ext_len = max_buffer_size or 1024*1024 + max_ext_len = max_buffer_size or 1024 * 1024 - self._max_buffer_size = max_buffer_size or 2**31-1 + self._max_buffer_size = max_buffer_size or 2 ** 31 - 1 if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") - self._read_size = read_size or min(self._max_buffer_size, 16*1024) + self._read_size = read_size or min(self._max_buffer_size, 16 * 1024) self._raw = bool(raw) self._strict_map_key = bool(strict_map_key) self._unicode_errors = unicode_errors @@ -310,26 +347,27 @@ class Unpacker(object): self._stream_offset = 0 if list_hook is not None and not callable(list_hook): - raise TypeError('`list_hook` is not callable') + raise TypeError("`list_hook` is not callable") if object_hook is not None and not callable(object_hook): - raise TypeError('`object_hook` is not callable') + raise TypeError("`object_hook` is not callable") if object_pairs_hook is not None and not callable(object_pairs_hook): - raise TypeError('`object_pairs_hook` is not callable') + raise TypeError("`object_pairs_hook` is not callable") if object_hook is not None and object_pairs_hook is not None: - raise TypeError("object_pairs_hook and object_hook are mutually " - "exclusive") + raise TypeError( + "object_pairs_hook and object_hook are mutually " "exclusive" + ) if not callable(ext_hook): raise TypeError("`ext_hook` is not callable") def feed(self, next_bytes): assert self._feeding view = _get_data_from_buffer(next_bytes) - if (len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size): + if len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size: raise BufferFull # Strip buffer before checkpoint before reading file. if self._buf_checkpoint > 0: - del self._buffer[:self._buf_checkpoint] + del self._buffer[: self._buf_checkpoint] self._buff_i -= self._buf_checkpoint self._buf_checkpoint = 0 @@ -345,7 +383,7 @@ class Unpacker(object): return self._buff_i < len(self._buffer) def _get_extradata(self): - return self._buffer[self._buff_i:] + return self._buffer[self._buff_i :] def read_bytes(self, n): return self._read(n) @@ -354,8 +392,8 @@ class Unpacker(object): # (int) -> bytearray self._reserve(n) i = self._buff_i - self._buff_i = i+n - return self._buffer[i:i+n] + self._buff_i = i + n + return self._buffer[i : i + n] def _reserve(self, n): remain_bytes = len(self._buffer) - self._buff_i - n @@ -370,7 +408,7 @@ class Unpacker(object): # Strip buffer before checkpoint before reading file. if self._buf_checkpoint > 0: - del self._buffer[:self._buf_checkpoint] + del self._buffer[: self._buf_checkpoint] self._buff_i -= self._buf_checkpoint self._buf_checkpoint = 0 @@ -399,7 +437,7 @@ class Unpacker(object): if b & 0b10000000 == 0: obj = b elif b & 0b11100000 == 0b11100000: - obj = -1 - (b ^ 0xff) + obj = -1 - (b ^ 0xFF) elif b & 0b11100000 == 0b10100000: n = b & 0b00011111 typ = TYPE_RAW @@ -416,13 +454,13 @@ class Unpacker(object): typ = TYPE_MAP if n > self._max_map_len: raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) - elif b == 0xc0: + elif b == 0xC0: obj = None - elif b == 0xc2: + elif b == 0xC2: obj = False - elif b == 0xc3: + elif b == 0xC3: obj = True - elif b == 0xc4: + elif b == 0xC4: typ = TYPE_BIN self._reserve(1) n = self._buffer[self._buff_i] @@ -430,7 +468,7 @@ class Unpacker(object): if n > self._max_bin_len: raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._read(n) - elif b == 0xc5: + elif b == 0xC5: typ = TYPE_BIN self._reserve(2) n = _unpack_from(">H", self._buffer, self._buff_i)[0] @@ -438,7 +476,7 @@ class Unpacker(object): if n > self._max_bin_len: raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._read(n) - elif b == 0xc6: + elif b == 0xC6: typ = TYPE_BIN self._reserve(4) n = _unpack_from(">I", self._buffer, self._buff_i)[0] @@ -446,106 +484,106 @@ class Unpacker(object): if n > self._max_bin_len: raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._read(n) - elif b == 0xc7: # ext 8 + elif b == 0xC7: # ext 8 typ = TYPE_EXT self._reserve(2) - L, n = _unpack_from('Bb', self._buffer, self._buff_i) + L, n = _unpack_from("Bb", self._buffer, self._buff_i) self._buff_i += 2 if L > self._max_ext_len: raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) - elif b == 0xc8: # ext 16 + elif b == 0xC8: # ext 16 typ = TYPE_EXT self._reserve(3) - L, n = _unpack_from('>Hb', self._buffer, self._buff_i) + L, n = _unpack_from(">Hb", self._buffer, self._buff_i) self._buff_i += 3 if L > self._max_ext_len: raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) - elif b == 0xc9: # ext 32 + elif b == 0xC9: # ext 32 typ = TYPE_EXT self._reserve(5) - L, n = _unpack_from('>Ib', self._buffer, self._buff_i) + L, n = _unpack_from(">Ib", self._buffer, self._buff_i) self._buff_i += 5 if L > self._max_ext_len: raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) - elif b == 0xca: + elif b == 0xCA: self._reserve(4) obj = _unpack_from(">f", self._buffer, self._buff_i)[0] self._buff_i += 4 - elif b == 0xcb: + elif b == 0xCB: self._reserve(8) obj = _unpack_from(">d", self._buffer, self._buff_i)[0] self._buff_i += 8 - elif b == 0xcc: + elif b == 0xCC: self._reserve(1) obj = self._buffer[self._buff_i] self._buff_i += 1 - elif b == 0xcd: + elif b == 0xCD: self._reserve(2) obj = _unpack_from(">H", self._buffer, self._buff_i)[0] self._buff_i += 2 - elif b == 0xce: + elif b == 0xCE: self._reserve(4) obj = _unpack_from(">I", self._buffer, self._buff_i)[0] self._buff_i += 4 - elif b == 0xcf: + elif b == 0xCF: self._reserve(8) obj = _unpack_from(">Q", self._buffer, self._buff_i)[0] self._buff_i += 8 - elif b == 0xd0: + elif b == 0xD0: self._reserve(1) obj = _unpack_from("b", self._buffer, self._buff_i)[0] self._buff_i += 1 - elif b == 0xd1: + elif b == 0xD1: self._reserve(2) obj = _unpack_from(">h", self._buffer, self._buff_i)[0] self._buff_i += 2 - elif b == 0xd2: + elif b == 0xD2: self._reserve(4) obj = _unpack_from(">i", self._buffer, self._buff_i)[0] self._buff_i += 4 - elif b == 0xd3: + elif b == 0xD3: self._reserve(8) obj = _unpack_from(">q", self._buffer, self._buff_i)[0] self._buff_i += 8 - elif b == 0xd4: # fixext 1 + elif b == 0xD4: # fixext 1 typ = TYPE_EXT if self._max_ext_len < 1: raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) self._reserve(2) n, obj = _unpack_from("b1s", self._buffer, self._buff_i) self._buff_i += 2 - elif b == 0xd5: # fixext 2 + elif b == 0xD5: # fixext 2 typ = TYPE_EXT if self._max_ext_len < 2: raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) self._reserve(3) n, obj = _unpack_from("b2s", self._buffer, self._buff_i) self._buff_i += 3 - elif b == 0xd6: # fixext 4 + elif b == 0xD6: # fixext 4 typ = TYPE_EXT if self._max_ext_len < 4: raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) self._reserve(5) n, obj = _unpack_from("b4s", self._buffer, self._buff_i) self._buff_i += 5 - elif b == 0xd7: # fixext 8 + elif b == 0xD7: # fixext 8 typ = TYPE_EXT if self._max_ext_len < 8: raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) self._reserve(9) n, obj = _unpack_from("b8s", self._buffer, self._buff_i) self._buff_i += 9 - elif b == 0xd8: # fixext 16 + elif b == 0xD8: # fixext 16 typ = TYPE_EXT if self._max_ext_len < 16: raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) self._reserve(17) n, obj = _unpack_from("b16s", self._buffer, self._buff_i) self._buff_i += 17 - elif b == 0xd9: + elif b == 0xD9: typ = TYPE_RAW self._reserve(1) n = self._buffer[self._buff_i] @@ -553,46 +591,46 @@ class Unpacker(object): if n > self._max_str_len: raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) - elif b == 0xda: + elif b == 0xDA: typ = TYPE_RAW self._reserve(2) - n, = _unpack_from(">H", self._buffer, self._buff_i) + (n,) = _unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_str_len: raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) - elif b == 0xdb: + elif b == 0xDB: typ = TYPE_RAW self._reserve(4) - n, = _unpack_from(">I", self._buffer, self._buff_i) + (n,) = _unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_str_len: raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) - elif b == 0xdc: + elif b == 0xDC: typ = TYPE_ARRAY self._reserve(2) - n, = _unpack_from(">H", self._buffer, self._buff_i) + (n,) = _unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_array_len: raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) - elif b == 0xdd: + elif b == 0xDD: typ = TYPE_ARRAY self._reserve(4) - n, = _unpack_from(">I", self._buffer, self._buff_i) + (n,) = _unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_array_len: raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) - elif b == 0xde: + elif b == 0xDE: self._reserve(2) - n, = _unpack_from(">H", self._buffer, self._buff_i) + (n,) = _unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_map_len: raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP - elif b == 0xdf: + elif b == 0xDF: self._reserve(4) - n, = _unpack_from(">I", self._buffer, self._buff_i) + (n,) = _unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_map_len: raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) @@ -635,15 +673,17 @@ class Unpacker(object): return if self._object_pairs_hook is not None: ret = self._object_pairs_hook( - (self._unpack(EX_CONSTRUCT), - self._unpack(EX_CONSTRUCT)) - for _ in xrange(n)) + (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT)) + for _ in xrange(n) + ) else: ret = {} for _ in xrange(n): key = self._unpack(EX_CONSTRUCT) if self._strict_map_key and type(key) not in (unicode, bytes): - raise ValueError("%s is not allowed for map key" % str(type(key))) + raise ValueError( + "%s is not allowed for map key" % str(type(key)) + ) if not PY2 and type(key) is str: key = sys.intern(key) ret[key] = self._unpack(EX_CONSTRUCT) @@ -656,7 +696,7 @@ class Unpacker(object): if self._raw: obj = bytes(obj) else: - obj = obj.decode('utf_8', self._unicode_errors) + obj = obj.decode("utf_8", self._unicode_errors) return obj if typ == TYPE_EXT: return self._ext_hook(n, bytes(obj)) @@ -746,9 +786,16 @@ class Packer(object): The error handler for encoding unicode. (default: 'strict') DO NOT USE THIS!! This option is kept for very specific usage. """ - def __init__(self, default=None, unicode_errors=None, - use_single_float=False, autoreset=True, use_bin_type=False, - strict_types=False): + + def __init__( + self, + default=None, + unicode_errors=None, + use_single_float=False, + autoreset=True, + use_bin_type=False, + strict_types=False, + ): self._strict_types = strict_types self._use_float = use_single_float self._autoreset = autoreset @@ -760,8 +807,13 @@ class Packer(object): raise TypeError("default must be callable") self._default = default - def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, - check=isinstance, check_type_strict=_check_type_strict): + def _pack( + self, + obj, + nest_limit=DEFAULT_RECURSE_LIMIT, + check=isinstance, + check_type_strict=_check_type_strict, + ): default_used = False if self._strict_types: check = check_type_strict @@ -782,22 +834,22 @@ class Packer(object): return self._buffer.write(struct.pack("B", obj)) if -0x20 <= obj < 0: return self._buffer.write(struct.pack("b", obj)) - if 0x80 <= obj <= 0xff: - return self._buffer.write(struct.pack("BB", 0xcc, obj)) + if 0x80 <= obj <= 0xFF: + return self._buffer.write(struct.pack("BB", 0xCC, obj)) if -0x80 <= obj < 0: - return self._buffer.write(struct.pack(">Bb", 0xd0, obj)) - if 0xff < obj <= 0xffff: - return self._buffer.write(struct.pack(">BH", 0xcd, obj)) + return self._buffer.write(struct.pack(">Bb", 0xD0, obj)) + if 0xFF < obj <= 0xFFFF: + return self._buffer.write(struct.pack(">BH", 0xCD, obj)) if -0x8000 <= obj < -0x80: - return self._buffer.write(struct.pack(">Bh", 0xd1, obj)) - if 0xffff < obj <= 0xffffffff: - return self._buffer.write(struct.pack(">BI", 0xce, obj)) + return self._buffer.write(struct.pack(">Bh", 0xD1, obj)) + if 0xFFFF < obj <= 0xFFFFFFFF: + return self._buffer.write(struct.pack(">BI", 0xCE, obj)) if -0x80000000 <= obj < -0x8000: - return self._buffer.write(struct.pack(">Bi", 0xd2, obj)) - if 0xffffffff < obj <= 0xffffffffffffffff: - return self._buffer.write(struct.pack(">BQ", 0xcf, obj)) + return self._buffer.write(struct.pack(">Bi", 0xD2, obj)) + if 0xFFFFFFFF < obj <= 0xFFFFFFFFFFFFFFFF: + return self._buffer.write(struct.pack(">BQ", 0xCF, obj)) if -0x8000000000000000 <= obj < -0x80000000: - return self._buffer.write(struct.pack(">Bq", 0xd3, obj)) + return self._buffer.write(struct.pack(">Bq", 0xD3, obj)) if not default_used and self._default is not None: obj = self._default(obj) default_used = True @@ -805,27 +857,27 @@ class Packer(object): raise OverflowError("Integer value out of range") if check(obj, (bytes, bytearray)): n = len(obj) - if n >= 2**32: + if n >= 2 ** 32: raise ValueError("%s is too large" % type(obj).__name__) self._pack_bin_header(n) return self._buffer.write(obj) if check(obj, unicode): obj = obj.encode("utf-8", self._unicode_errors) n = len(obj) - if n >= 2**32: + if n >= 2 ** 32: raise ValueError("String is too large") self._pack_raw_header(n) return self._buffer.write(obj) if check(obj, memoryview): n = len(obj) * obj.itemsize - if n >= 2**32: + if n >= 2 ** 32: raise ValueError("Memoryview is too large") self._pack_bin_header(n) return self._buffer.write(obj) if check(obj, float): if self._use_float: - return self._buffer.write(struct.pack(">Bf", 0xca, obj)) - return self._buffer.write(struct.pack(">Bd", 0xcb, obj)) + return self._buffer.write(struct.pack(">Bf", 0xCA, obj)) + return self._buffer.write(struct.pack(">Bd", 0xCB, obj)) if check(obj, (ExtType, Timestamp)): if check(obj, Timestamp): code = -1 @@ -837,21 +889,21 @@ class Packer(object): assert isinstance(data, bytes) L = len(data) if L == 1: - self._buffer.write(b'\xd4') + self._buffer.write(b"\xd4") elif L == 2: - self._buffer.write(b'\xd5') + self._buffer.write(b"\xd5") elif L == 4: - self._buffer.write(b'\xd6') + self._buffer.write(b"\xd6") elif L == 8: - self._buffer.write(b'\xd7') + self._buffer.write(b"\xd7") elif L == 16: - self._buffer.write(b'\xd8') - elif L <= 0xff: - self._buffer.write(struct.pack(">BB", 0xc7, L)) - elif L <= 0xffff: - self._buffer.write(struct.pack(">BH", 0xc8, L)) + self._buffer.write(b"\xd8") + elif L <= 0xFF: + self._buffer.write(struct.pack(">BB", 0xC7, L)) + elif L <= 0xFFFF: + self._buffer.write(struct.pack(">BH", 0xC8, L)) else: - self._buffer.write(struct.pack(">BI", 0xc9, L)) + self._buffer.write(struct.pack(">BI", 0xC9, L)) self._buffer.write(struct.pack("b", code)) self._buffer.write(data) return @@ -862,13 +914,14 @@ class Packer(object): self._pack(obj[i], nest_limit - 1) return if check(obj, dict): - return self._pack_map_pairs(len(obj), dict_iteritems(obj), - nest_limit - 1) + return self._pack_map_pairs( + len(obj), dict_iteritems(obj), nest_limit - 1 + ) if not default_used and self._default is not None: obj = self._default(obj) default_used = 1 continue - raise TypeError("Cannot serialize %r" % (obj, )) + raise TypeError("Cannot serialize %r" % (obj,)) def pack(self, obj): try: @@ -889,7 +942,7 @@ class Packer(object): return ret def pack_array_header(self, n): - if n >= 2**32: + if n >= 2 ** 32: raise ValueError self._pack_array_header(n) if self._autoreset: @@ -898,7 +951,7 @@ class Packer(object): return ret def pack_map_header(self, n): - if n >= 2**32: + if n >= 2 ** 32: raise ValueError self._pack_map_header(n) if self._autoreset: @@ -914,43 +967,43 @@ class Packer(object): if not isinstance(data, bytes): raise TypeError("data must have bytes type") L = len(data) - if L > 0xffffffff: + if L > 0xFFFFFFFF: raise ValueError("Too large data") if L == 1: - self._buffer.write(b'\xd4') + self._buffer.write(b"\xd4") elif L == 2: - self._buffer.write(b'\xd5') + self._buffer.write(b"\xd5") elif L == 4: - self._buffer.write(b'\xd6') + self._buffer.write(b"\xd6") elif L == 8: - self._buffer.write(b'\xd7') + self._buffer.write(b"\xd7") elif L == 16: - self._buffer.write(b'\xd8') - elif L <= 0xff: - self._buffer.write(b'\xc7' + struct.pack('B', L)) - elif L <= 0xffff: - self._buffer.write(b'\xc8' + struct.pack('>H', L)) + self._buffer.write(b"\xd8") + elif L <= 0xFF: + self._buffer.write(b"\xc7" + struct.pack("B", L)) + elif L <= 0xFFFF: + self._buffer.write(b"\xc8" + struct.pack(">H", L)) else: - self._buffer.write(b'\xc9' + struct.pack('>I', L)) - self._buffer.write(struct.pack('B', typecode)) + self._buffer.write(b"\xc9" + struct.pack(">I", L)) + self._buffer.write(struct.pack("B", typecode)) self._buffer.write(data) def _pack_array_header(self, n): - if n <= 0x0f: - return self._buffer.write(struct.pack('B', 0x90 + n)) - if n <= 0xffff: - return self._buffer.write(struct.pack(">BH", 0xdc, n)) - if n <= 0xffffffff: - return self._buffer.write(struct.pack(">BI", 0xdd, n)) + if n <= 0x0F: + return self._buffer.write(struct.pack("B", 0x90 + n)) + if n <= 0xFFFF: + return self._buffer.write(struct.pack(">BH", 0xDC, n)) + if n <= 0xFFFFFFFF: + return self._buffer.write(struct.pack(">BI", 0xDD, n)) raise ValueError("Array is too large") def _pack_map_header(self, n): - if n <= 0x0f: - return self._buffer.write(struct.pack('B', 0x80 + n)) - if n <= 0xffff: - return self._buffer.write(struct.pack(">BH", 0xde, n)) - if n <= 0xffffffff: - return self._buffer.write(struct.pack(">BI", 0xdf, n)) + if n <= 0x0F: + return self._buffer.write(struct.pack("B", 0x80 + n)) + if n <= 0xFFFF: + return self._buffer.write(struct.pack(">BH", 0xDE, n)) + if n <= 0xFFFFFFFF: + return self._buffer.write(struct.pack(">BI", 0xDF, n)) raise ValueError("Dict is too large") def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT): @@ -960,28 +1013,28 @@ class Packer(object): self._pack(v, nest_limit - 1) def _pack_raw_header(self, n): - if n <= 0x1f: - self._buffer.write(struct.pack('B', 0xa0 + n)) - elif self._use_bin_type and n <= 0xff: - self._buffer.write(struct.pack('>BB', 0xd9, n)) - elif n <= 0xffff: - self._buffer.write(struct.pack(">BH", 0xda, n)) - elif n <= 0xffffffff: - self._buffer.write(struct.pack(">BI", 0xdb, n)) + if n <= 0x1F: + self._buffer.write(struct.pack("B", 0xA0 + n)) + elif self._use_bin_type and n <= 0xFF: + self._buffer.write(struct.pack(">BB", 0xD9, n)) + elif n <= 0xFFFF: + self._buffer.write(struct.pack(">BH", 0xDA, n)) + elif n <= 0xFFFFFFFF: + self._buffer.write(struct.pack(">BI", 0xDB, n)) else: - raise ValueError('Raw is too large') + raise ValueError("Raw is too large") def _pack_bin_header(self, n): if not self._use_bin_type: return self._pack_raw_header(n) - elif n <= 0xff: - return self._buffer.write(struct.pack('>BB', 0xc4, n)) - elif n <= 0xffff: - return self._buffer.write(struct.pack(">BH", 0xc5, n)) - elif n <= 0xffffffff: - return self._buffer.write(struct.pack(">BI", 0xc6, n)) + elif n <= 0xFF: + return self._buffer.write(struct.pack(">BB", 0xC4, n)) + elif n <= 0xFFFF: + return self._buffer.write(struct.pack(">BH", 0xC5, n)) + elif n <= 0xFFFFFFFF: + return self._buffer.write(struct.pack(">BI", 0xC6, n)) else: - raise ValueError('Bin is too large') + raise ValueError("Bin is too large") def bytes(self): """Return internal buffer contents as bytes object""" From 10e5e39ff9739fa3ce589ad9d451260be0f3842c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 5 Dec 2019 18:51:45 +0900 Subject: [PATCH 1009/1172] blacken test --- test/test_buffer.py | 17 +++--- test/test_case.py | 116 +++++++++++++++++++++++++------------- test/test_extension.py | 55 ++++++++++-------- test/test_format.py | 92 ++++++++++++++++++------------ test/test_limits.py | 46 ++++++++------- test/test_memoryview.py | 47 ++++++++-------- test/test_newspec.py | 60 ++++++++++---------- test/test_obj.py | 42 +++++++++----- test/test_pack.py | 102 ++++++++++++++++++++++++--------- test/test_read_size.py | 49 ++++++++-------- test/test_seq.py | 11 ++-- test/test_sequnpack.py | 121 ++++++++++++++++++++++------------------ test/test_stricttype.py | 29 +++++----- test/test_subtype.py | 7 ++- test/test_timestamp.py | 14 ++--- test/test_unpack.py | 27 +++++---- 16 files changed, 501 insertions(+), 334 deletions(-) diff --git a/test/test_buffer.py b/test/test_buffer.py index d723e8d..64fbdef 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -6,27 +6,28 @@ from msgpack import packb, unpackb def test_unpack_buffer(): from array import array - buf = array('b') + + buf = array("b") try: - buf.frombytes(packb((b'foo', b'bar'))) + buf.frombytes(packb((b"foo", b"bar"))) except AttributeError: # PY2 - buf.fromstring(packb((b'foo', b'bar'))) + buf.fromstring(packb((b"foo", b"bar"))) obj = unpackb(buf, use_list=1) - assert [b'foo', b'bar'] == obj + assert [b"foo", b"bar"] == obj def test_unpack_bytearray(): - buf = bytearray(packb(('foo', 'bar'))) + buf = bytearray(packb(("foo", "bar"))) obj = unpackb(buf, use_list=1) - assert [b'foo', b'bar'] == obj + assert [b"foo", b"bar"] == obj expected_type = bytes assert all(type(s) == expected_type for s in obj) def test_unpack_memoryview(): - buf = bytearray(packb(('foo', 'bar'))) + buf = bytearray(packb(("foo", "bar"))) view = memoryview(buf) obj = unpackb(view, use_list=1) - assert [b'foo', b'bar'] == obj + assert [b"foo", b"bar"] == obj expected_type = bytes assert all(type(s) == expected_type for s in obj) diff --git a/test/test_case.py b/test/test_case.py index 5a4bb6c..3bc1b26 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -6,97 +6,133 @@ from msgpack import packb, unpackb def check(length, obj): v = packb(obj) - assert len(v) == length, \ - "%r length should be %r but get %r" % (obj, length, len(v)) + assert len(v) == length, "%r length should be %r but get %r" % (obj, length, len(v)) assert unpackb(v, use_list=0) == obj + def test_1(): - for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1, - -((1<<5)-1), -(1<<5)]: + for o in [ + None, + True, + False, + 0, + 1, + (1 << 6), + (1 << 7) - 1, + -1, + -((1 << 5) - 1), + -(1 << 5), + ]: check(1, o) + def test_2(): - for o in [1 << 7, (1 << 8) - 1, - -((1<<5)+1), -(1<<7) - ]: + for o in [1 << 7, (1 << 8) - 1, -((1 << 5) + 1), -(1 << 7)]: check(2, o) + def test_3(): - for o in [1 << 8, (1 << 16) - 1, - -((1<<7)+1), -(1<<15)]: + for o in [1 << 8, (1 << 16) - 1, -((1 << 7) + 1), -(1 << 15)]: check(3, o) + def test_5(): - for o in [1 << 16, (1 << 32) - 1, - -((1<<15)+1), -(1<<31)]: + for o in [1 << 16, (1 << 32) - 1, -((1 << 15) + 1), -(1 << 31)]: check(5, o) + def test_9(): - for o in [1 << 32, (1 << 64) - 1, - -((1<<31)+1), -(1<<63), - 1.0, 0.1, -0.1, -1.0]: + for o in [ + 1 << 32, + (1 << 64) - 1, + -((1 << 31) + 1), + -(1 << 63), + 1.0, + 0.1, + -0.1, + -1.0, + ]: check(9, o) def check_raw(overhead, num): check(num + overhead, b" " * num) + def test_fixraw(): check_raw(1, 0) - check_raw(1, (1<<5) - 1) + check_raw(1, (1 << 5) - 1) + def test_raw16(): - check_raw(3, 1<<5) - check_raw(3, (1<<16) - 1) + check_raw(3, 1 << 5) + check_raw(3, (1 << 16) - 1) + def test_raw32(): - check_raw(5, 1<<16) + check_raw(5, 1 << 16) def check_array(overhead, num): check(num + overhead, (None,) * num) + def test_fixarray(): check_array(1, 0) check_array(1, (1 << 4) - 1) + def test_array16(): check_array(3, 1 << 4) - check_array(3, (1<<16)-1) + check_array(3, (1 << 16) - 1) + def test_array32(): - check_array(5, (1<<16)) + check_array(5, (1 << 16)) def match(obj, buf): assert packb(obj) == buf assert unpackb(buf, use_list=0) == obj + def test_match(): cases = [ - (None, b'\xc0'), - (False, b'\xc2'), - (True, b'\xc3'), - (0, b'\x00'), - (127, b'\x7f'), - (128, b'\xcc\x80'), - (256, b'\xcd\x01\x00'), - (-1, b'\xff'), - (-33, b'\xd0\xdf'), - (-129, b'\xd1\xff\x7f'), - ({1:1}, b'\x81\x01\x01'), + (None, b"\xc0"), + (False, b"\xc2"), + (True, b"\xc3"), + (0, b"\x00"), + (127, b"\x7f"), + (128, b"\xcc\x80"), + (256, b"\xcd\x01\x00"), + (-1, b"\xff"), + (-33, b"\xd0\xdf"), + (-129, b"\xd1\xff\x7f"), + ({1: 1}, b"\x81\x01\x01"), (1.0, b"\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"), - ((), b'\x90'), - (tuple(range(15)),b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), - (tuple(range(16)),b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), - ({}, b'\x80'), - (dict([(x,x) for x in range(15)]), b'\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'), - (dict([(x,x) for x in range(16)]), b'\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'), - ] + ((), b"\x90"), + ( + tuple(range(15)), + b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e", + ), + ( + tuple(range(16)), + b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + ), + ({}, b"\x80"), + ( + dict([(x, x) for x in range(15)]), + b"\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e", + ), + ( + dict([(x, x) for x in range(16)]), + b"\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f", + ), + ] for v, p in cases: match(v, p) -def test_unicode(): - assert unpackb(packb('foobar'), use_list=1) == b'foobar' +def test_unicode(): + assert unpackb(packb("foobar"), use_list=1) == b"foobar" diff --git a/test/test_extension.py b/test/test_extension.py index 8aa0cbb..6b36575 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -9,37 +9,41 @@ def test_pack_ext_type(): packer = msgpack.Packer() packer.pack_ext_type(0x42, s) return packer.bytes() - assert p(b'A') == b'\xd4\x42A' # fixext 1 - assert p(b'AB') == b'\xd5\x42AB' # fixext 2 - assert p(b'ABCD') == b'\xd6\x42ABCD' # fixext 4 - assert p(b'ABCDEFGH') == b'\xd7\x42ABCDEFGH' # fixext 8 - assert p(b'A'*16) == b'\xd8\x42' + b'A'*16 # fixext 16 - assert p(b'ABC') == b'\xc7\x03\x42ABC' # ext 8 - assert p(b'A'*0x0123) == b'\xc8\x01\x23\x42' + b'A'*0x0123 # ext 16 - assert p(b'A'*0x00012345) == b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345 # ext 32 + + assert p(b"A") == b"\xd4\x42A" # fixext 1 + assert p(b"AB") == b"\xd5\x42AB" # fixext 2 + assert p(b"ABCD") == b"\xd6\x42ABCD" # fixext 4 + assert p(b"ABCDEFGH") == b"\xd7\x42ABCDEFGH" # fixext 8 + assert p(b"A" * 16) == b"\xd8\x42" + b"A" * 16 # fixext 16 + assert p(b"ABC") == b"\xc7\x03\x42ABC" # ext 8 + assert p(b"A" * 0x0123) == b"\xc8\x01\x23\x42" + b"A" * 0x0123 # ext 16 + assert ( + p(b"A" * 0x00012345) == b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345 + ) # ext 32 def test_unpack_ext_type(): def check(b, expected): assert msgpack.unpackb(b) == expected - check(b'\xd4\x42A', ExtType(0x42, b'A')) # fixext 1 - check(b'\xd5\x42AB', ExtType(0x42, b'AB')) # fixext 2 - check(b'\xd6\x42ABCD', ExtType(0x42, b'ABCD')) # fixext 4 - check(b'\xd7\x42ABCDEFGH', ExtType(0x42, b'ABCDEFGH')) # fixext 8 - check(b'\xd8\x42' + b'A'*16, ExtType(0x42, b'A'*16)) # fixext 16 - check(b'\xc7\x03\x42ABC', ExtType(0x42, b'ABC')) # ext 8 - check(b'\xc8\x01\x23\x42' + b'A'*0x0123, - ExtType(0x42, b'A'*0x0123)) # ext 16 - check(b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345, - ExtType(0x42, b'A'*0x00012345)) # ext 32 + check(b"\xd4\x42A", ExtType(0x42, b"A")) # fixext 1 + check(b"\xd5\x42AB", ExtType(0x42, b"AB")) # fixext 2 + check(b"\xd6\x42ABCD", ExtType(0x42, b"ABCD")) # fixext 4 + check(b"\xd7\x42ABCDEFGH", ExtType(0x42, b"ABCDEFGH")) # fixext 8 + check(b"\xd8\x42" + b"A" * 16, ExtType(0x42, b"A" * 16)) # fixext 16 + check(b"\xc7\x03\x42ABC", ExtType(0x42, b"ABC")) # ext 8 + check(b"\xc8\x01\x23\x42" + b"A" * 0x0123, ExtType(0x42, b"A" * 0x0123)) # ext 16 + check( + b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345, + ExtType(0x42, b"A" * 0x00012345), + ) # ext 32 def test_extension_type(): def default(obj): - print('default called', obj) + print("default called", obj) if isinstance(obj, array.array): - typecode = 123 # application specific typecode + typecode = 123 # application specific typecode try: data = obj.tobytes() except AttributeError: @@ -48,24 +52,27 @@ def test_extension_type(): raise TypeError("Unknown type object %r" % (obj,)) def ext_hook(code, data): - print('ext_hook called', code, data) + print("ext_hook called", code, data) assert code == 123 - obj = array.array('d') + obj = array.array("d") try: obj.frombytes(data) except AttributeError: # PY2 obj.fromstring(data) return obj - obj = [42, b'hello', array.array('d', [1.1, 2.2, 3.3])] + obj = [42, b"hello", array.array("d", [1.1, 2.2, 3.3])] s = msgpack.packb(obj, default=default) obj2 = msgpack.unpackb(s, ext_hook=ext_hook) assert obj == obj2 + import sys -if sys.version > '3': + +if sys.version > "3": long = int + def test_overriding_hooks(): def default(obj): if isinstance(obj, long): diff --git a/test/test_format.py b/test/test_format.py index 5fec0c3..c2cdfbd 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -3,68 +3,88 @@ from msgpack import unpackb + def check(src, should, use_list=0): assert unpackb(src, use_list=use_list) == should + def testSimpleValue(): - check(b"\x93\xc0\xc2\xc3", - (None, False, True,)) + check(b"\x93\xc0\xc2\xc3", (None, False, True,)) + def testFixnum(): - check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", - ((0,64,127,), (-32,-16,-1,),) - ) + check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", ((0, 64, 127,), (-32, -16, -1,),)) + def testFixArray(): - check(b"\x92\x90\x91\x91\xc0", - ((),((None,),),), - ) + check( + b"\x92\x90\x91\x91\xc0", ((), ((None,),),), + ) + def testFixRaw(): - check(b"\x94\xa0\xa1a\xa2bc\xa3def", - (b"", b"a", b"bc", b"def",), - ) + check( + b"\x94\xa0\xa1a\xa2bc\xa3def", (b"", b"a", b"bc", b"def",), + ) + def testFixMap(): check( - b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", - {False: {None: None}, True:{None:{}}}, - ) + b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", + {False: {None: None}, True: {None: {}}}, + ) + def testUnsignedInt(): check( - b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" - b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" - b"\xce\xff\xff\xff\xff", - (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,), - ) + b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" + b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" + b"\xce\xff\xff\xff\xff", + (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,), + ) + def testSignedInt(): - check(b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" - b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" - b"\xd2\xff\xff\xff\xff", - (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,)) + check( + b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" + b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" + b"\xd2\xff\xff\xff\xff", + (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,), + ) + def testRaw(): - check(b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" + check( + b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", - (b"", b"a", b"ab", b"", b"a", b"ab")) + (b"", b"a", b"ab", b"", b"a", b"ab"), + ) + def testArray(): - check(b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" + check( + b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" b"\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02" b"\xc2\xc3", - ((), (None,), (False,True), (), (None,), (False,True)) - ) + ((), (None,), (False, True), (), (None,), (False, True)), + ) + def testMap(): check( b"\x96" - b"\xde\x00\x00" - b"\xde\x00\x01\xc0\xc2" - b"\xde\x00\x02\xc0\xc2\xc3\xc2" - b"\xdf\x00\x00\x00\x00" - b"\xdf\x00\x00\x00\x01\xc0\xc2" - b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", - ({}, {None: False}, {True: False, None: False}, {}, - {None: False}, {True: False, None: False})) + b"\xde\x00\x00" + b"\xde\x00\x01\xc0\xc2" + b"\xde\x00\x02\xc0\xc2\xc3\xc2" + b"\xdf\x00\x00\x00\x00" + b"\xdf\x00\x00\x00\x01\xc0\xc2" + b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", + ( + {}, + {None: False}, + {True: False, None: False}, + {}, + {None: False}, + {True: False, None: False}, + ), + ) diff --git a/test/test_limits.py b/test/test_limits.py index 8c7606f..6e85030 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -4,8 +4,14 @@ from __future__ import absolute_import, division, print_function, unicode_litera import pytest from msgpack import ( - packb, unpackb, Packer, Unpacker, ExtType, - PackOverflowError, PackValueError, UnpackValueError, + packb, + unpackb, + Packer, + Unpacker, + ExtType, + PackOverflowError, + PackValueError, + UnpackValueError, ) @@ -13,30 +19,30 @@ def test_integer(): x = -(2 ** 63) assert unpackb(packb(x)) == x with pytest.raises(PackOverflowError): - packb(x-1) + packb(x - 1) x = 2 ** 64 - 1 assert unpackb(packb(x)) == x with pytest.raises(PackOverflowError): - packb(x+1) + packb(x + 1) def test_array_header(): packer = Packer() - packer.pack_array_header(2**32-1) + packer.pack_array_header(2 ** 32 - 1) with pytest.raises(PackValueError): - packer.pack_array_header(2**32) + packer.pack_array_header(2 ** 32) def test_map_header(): packer = Packer() - packer.pack_map_header(2**32-1) + packer.pack_map_header(2 ** 32 - 1) with pytest.raises(PackValueError): - packer.pack_array_header(2**32) + packer.pack_array_header(2 ** 32) def test_max_str_len(): - d = 'x' * 3 + d = "x" * 3 packed = packb(d) unpacker = Unpacker(max_str_len=3, raw=False) @@ -50,7 +56,7 @@ def test_max_str_len(): def test_max_bin_len(): - d = b'x' * 3 + d = b"x" * 3 packed = packb(d, use_bin_type=True) unpacker = Unpacker(max_bin_len=3) @@ -64,7 +70,7 @@ def test_max_bin_len(): def test_max_array_len(): - d = [1,2,3] + d = [1, 2, 3] packed = packb(d) unpacker = Unpacker(max_array_len=3) @@ -107,8 +113,8 @@ def test_max_ext_len(): # PyPy fails following tests because of constant folding? # https://bugs.pypy.org/issue1721 -#@pytest.mark.skipif(True, reason="Requires very large memory.") -#def test_binary(): +# @pytest.mark.skipif(True, reason="Requires very large memory.") +# def test_binary(): # x = b'x' * (2**32 - 1) # assert unpackb(packb(x)) == x # del x @@ -117,8 +123,8 @@ def test_max_ext_len(): # packb(x) # # -#@pytest.mark.skipif(True, reason="Requires very large memory.") -#def test_string(): +# @pytest.mark.skipif(True, reason="Requires very large memory.") +# def test_string(): # x = 'x' * (2**32 - 1) # assert unpackb(packb(x)) == x # x += 'y' @@ -126,8 +132,8 @@ def test_max_ext_len(): # packb(x) # # -#@pytest.mark.skipif(True, reason="Requires very large memory.") -#def test_array(): +# @pytest.mark.skipif(True, reason="Requires very large memory.") +# def test_array(): # x = [0] * (2**32 - 1) # assert unpackb(packb(x)) == x # x.append(0) @@ -137,8 +143,9 @@ def test_max_ext_len(): # auto max len + def test_auto_max_array_len(): - packed = b'\xde\x00\x06zz' + packed = b"\xde\x00\x06zz" with pytest.raises(UnpackValueError): unpackb(packed, raw=False) @@ -147,9 +154,10 @@ def test_auto_max_array_len(): with pytest.raises(UnpackValueError): unpacker.unpack() + def test_auto_max_map_len(): # len(packed) == 6 -> max_map_len == 3 - packed = b'\xde\x00\x04zzz' + packed = b"\xde\x00\x04zzz" with pytest.raises(UnpackValueError): unpackb(packed, raw=False) diff --git a/test/test_memoryview.py b/test/test_memoryview.py index f6d74ed..e1b63b8 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -10,6 +10,7 @@ import sys # - array type only supports old buffer interface # - array.frombytes is not available, must use deprecated array.fromstring if sys.version_info[0] < 3: + def make_memoryview(obj): return memoryview(buffer(obj)) @@ -20,6 +21,8 @@ if sys.version_info[0] < 3: def get_data(a): return a.tostring() + + else: make_memoryview = memoryview @@ -49,64 +52,64 @@ def _runtest(format, nbytes, expected_header, expected_prefix, use_bin_type): # check packed header assert packed[:1] == expected_header # check packed length prefix, if any - assert packed[1:1+len(expected_prefix)] == expected_prefix + assert packed[1 : 1 + len(expected_prefix)] == expected_prefix # check packed data - assert packed[1+len(expected_prefix):] == original_data + assert packed[1 + len(expected_prefix) :] == original_data # check array unpacked correctly assert original_array == reconstructed_array def test_fixstr_from_byte(): - _runtest('B', 1, b'\xa1', b'', False) - _runtest('B', 31, b'\xbf', b'', False) + _runtest("B", 1, b"\xa1", b"", False) + _runtest("B", 31, b"\xbf", b"", False) def test_fixstr_from_float(): - _runtest('f', 4, b'\xa4', b'', False) - _runtest('f', 28, b'\xbc', b'', False) + _runtest("f", 4, b"\xa4", b"", False) + _runtest("f", 28, b"\xbc", b"", False) def test_str16_from_byte(): - _runtest('B', 2**8, b'\xda', b'\x01\x00', False) - _runtest('B', 2**16-1, b'\xda', b'\xff\xff', False) + _runtest("B", 2 ** 8, b"\xda", b"\x01\x00", False) + _runtest("B", 2 ** 16 - 1, b"\xda", b"\xff\xff", False) def test_str16_from_float(): - _runtest('f', 2**8, b'\xda', b'\x01\x00', False) - _runtest('f', 2**16-4, b'\xda', b'\xff\xfc', False) + _runtest("f", 2 ** 8, b"\xda", b"\x01\x00", False) + _runtest("f", 2 ** 16 - 4, b"\xda", b"\xff\xfc", False) def test_str32_from_byte(): - _runtest('B', 2**16, b'\xdb', b'\x00\x01\x00\x00', False) + _runtest("B", 2 ** 16, b"\xdb", b"\x00\x01\x00\x00", False) def test_str32_from_float(): - _runtest('f', 2**16, b'\xdb', b'\x00\x01\x00\x00', False) + _runtest("f", 2 ** 16, b"\xdb", b"\x00\x01\x00\x00", False) def test_bin8_from_byte(): - _runtest('B', 1, b'\xc4', b'\x01', True) - _runtest('B', 2**8-1, b'\xc4', b'\xff', True) + _runtest("B", 1, b"\xc4", b"\x01", True) + _runtest("B", 2 ** 8 - 1, b"\xc4", b"\xff", True) def test_bin8_from_float(): - _runtest('f', 4, b'\xc4', b'\x04', True) - _runtest('f', 2**8-4, b'\xc4', b'\xfc', True) + _runtest("f", 4, b"\xc4", b"\x04", True) + _runtest("f", 2 ** 8 - 4, b"\xc4", b"\xfc", True) def test_bin16_from_byte(): - _runtest('B', 2**8, b'\xc5', b'\x01\x00', True) - _runtest('B', 2**16-1, b'\xc5', b'\xff\xff', True) + _runtest("B", 2 ** 8, b"\xc5", b"\x01\x00", True) + _runtest("B", 2 ** 16 - 1, b"\xc5", b"\xff\xff", True) def test_bin16_from_float(): - _runtest('f', 2**8, b'\xc5', b'\x01\x00', True) - _runtest('f', 2**16-4, b'\xc5', b'\xff\xfc', True) + _runtest("f", 2 ** 8, b"\xc5", b"\x01\x00", True) + _runtest("f", 2 ** 16 - 4, b"\xc5", b"\xff\xfc", True) def test_bin32_from_byte(): - _runtest('B', 2**16, b'\xc6', b'\x00\x01\x00\x00', True) + _runtest("B", 2 ** 16, b"\xc6", b"\x00\x01\x00\x00", True) def test_bin32_from_float(): - _runtest('f', 2**16, b'\xc6', b'\x00\x01\x00\x00', True) + _runtest("f", 2 ** 16, b"\xc6", b"\x00\x01\x00\x00", True) diff --git a/test/test_newspec.py b/test/test_newspec.py index ab05029..f4f2a23 100644 --- a/test/test_newspec.py +++ b/test/test_newspec.py @@ -4,85 +4,87 @@ from msgpack import packb, unpackb, ExtType def test_str8(): - header = b'\xd9' - data = b'x' * 32 + header = b"\xd9" + data = b"x" * 32 b = packb(data.decode(), use_bin_type=True) assert len(b) == len(data) + 2 - assert b[0:2] == header + b'\x20' + assert b[0:2] == header + b"\x20" assert b[2:] == data assert unpackb(b) == data - data = b'x' * 255 + data = b"x" * 255 b = packb(data.decode(), use_bin_type=True) assert len(b) == len(data) + 2 - assert b[0:2] == header + b'\xff' + assert b[0:2] == header + b"\xff" assert b[2:] == data assert unpackb(b) == data def test_bin8(): - header = b'\xc4' - data = b'' + header = b"\xc4" + data = b"" b = packb(data, use_bin_type=True) assert len(b) == len(data) + 2 - assert b[0:2] == header + b'\x00' + assert b[0:2] == header + b"\x00" assert b[2:] == data assert unpackb(b) == data - data = b'x' * 255 + data = b"x" * 255 b = packb(data, use_bin_type=True) assert len(b) == len(data) + 2 - assert b[0:2] == header + b'\xff' + assert b[0:2] == header + b"\xff" assert b[2:] == data assert unpackb(b) == data def test_bin16(): - header = b'\xc5' - data = b'x' * 256 + header = b"\xc5" + data = b"x" * 256 b = packb(data, use_bin_type=True) assert len(b) == len(data) + 3 assert b[0:1] == header - assert b[1:3] == b'\x01\x00' + assert b[1:3] == b"\x01\x00" assert b[3:] == data assert unpackb(b) == data - data = b'x' * 65535 + data = b"x" * 65535 b = packb(data, use_bin_type=True) assert len(b) == len(data) + 3 assert b[0:1] == header - assert b[1:3] == b'\xff\xff' + assert b[1:3] == b"\xff\xff" assert b[3:] == data assert unpackb(b) == data def test_bin32(): - header = b'\xc6' - data = b'x' * 65536 + header = b"\xc6" + data = b"x" * 65536 b = packb(data, use_bin_type=True) assert len(b) == len(data) + 5 assert b[0:1] == header - assert b[1:5] == b'\x00\x01\x00\x00' + assert b[1:5] == b"\x00\x01\x00\x00" assert b[5:] == data assert unpackb(b) == data + def test_ext(): def check(ext, packed): assert packb(ext) == packed assert unpackb(packed) == ext - check(ExtType(0x42, b'Z'), b'\xd4\x42Z') # fixext 1 - check(ExtType(0x42, b'ZZ'), b'\xd5\x42ZZ') # fixext 2 - check(ExtType(0x42, b'Z'*4), b'\xd6\x42' + b'Z'*4) # fixext 4 - check(ExtType(0x42, b'Z'*8), b'\xd7\x42' + b'Z'*8) # fixext 8 - check(ExtType(0x42, b'Z'*16), b'\xd8\x42' + b'Z'*16) # fixext 16 + + check(ExtType(0x42, b"Z"), b"\xd4\x42Z") # fixext 1 + check(ExtType(0x42, b"ZZ"), b"\xd5\x42ZZ") # fixext 2 + check(ExtType(0x42, b"Z" * 4), b"\xd6\x42" + b"Z" * 4) # fixext 4 + check(ExtType(0x42, b"Z" * 8), b"\xd7\x42" + b"Z" * 8) # fixext 8 + check(ExtType(0x42, b"Z" * 16), b"\xd8\x42" + b"Z" * 16) # fixext 16 # ext 8 - check(ExtType(0x42, b''), b'\xc7\x00\x42') - check(ExtType(0x42, b'Z'*255), b'\xc7\xff\x42' + b'Z'*255) + check(ExtType(0x42, b""), b"\xc7\x00\x42") + check(ExtType(0x42, b"Z" * 255), b"\xc7\xff\x42" + b"Z" * 255) # ext 16 - check(ExtType(0x42, b'Z'*256), b'\xc8\x01\x00\x42' + b'Z'*256) - check(ExtType(0x42, b'Z'*0xffff), b'\xc8\xff\xff\x42' + b'Z'*0xffff) + check(ExtType(0x42, b"Z" * 256), b"\xc8\x01\x00\x42" + b"Z" * 256) + check(ExtType(0x42, b"Z" * 0xFFFF), b"\xc8\xff\xff\x42" + b"Z" * 0xFFFF) # ext 32 - check(ExtType(0x42, b'Z'*0x10000), b'\xc9\x00\x01\x00\x00\x42' + b'Z'*0x10000) + check(ExtType(0x42, b"Z" * 0x10000), b"\xc9\x00\x01\x00\x00\x42" + b"Z" * 0x10000) # needs large memory - #check(ExtType(0x42, b'Z'*0xffffffff), + # check(ExtType(0x42, b'Z'*0xffffffff), # b'\xc9\xff\xff\xff\xff\x42' + b'Z'*0xffffffff) diff --git a/test/test_obj.py b/test/test_obj.py index 390c1b6..0b99cea 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -4,64 +4,76 @@ from pytest import raises from msgpack import packb, unpackb + def _decode_complex(obj): - if b'__complex__' in obj: - return complex(obj[b'real'], obj[b'imag']) + if b"__complex__" in obj: + return complex(obj[b"real"], obj[b"imag"]) return obj + def _encode_complex(obj): if isinstance(obj, complex): - return {b'__complex__': True, b'real': 1, b'imag': 2} + return {b"__complex__": True, b"real": 1, b"imag": 2} return obj + def test_encode_hook(): - packed = packb([3, 1+2j], default=_encode_complex) + packed = packb([3, 1 + 2j], default=_encode_complex) unpacked = unpackb(packed, use_list=1) - assert unpacked[1] == {b'__complex__': True, b'real': 1, b'imag': 2} + assert unpacked[1] == {b"__complex__": True, b"real": 1, b"imag": 2} + def test_decode_hook(): - packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}]) + packed = packb([3, {b"__complex__": True, b"real": 1, b"imag": 2}]) unpacked = unpackb(packed, object_hook=_decode_complex, use_list=1) - assert unpacked[1] == 1+2j + assert unpacked[1] == 1 + 2j + def test_decode_pairs_hook(): packed = packb([3, {1: 2, 3: 4}]) prod_sum = 1 * 2 + 3 * 4 - unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l), use_list=1) + unpacked = unpackb( + packed, object_pairs_hook=lambda l: sum(k * v for k, v in l), use_list=1 + ) assert unpacked[1] == prod_sum + def test_only_one_obj_hook(): with raises(TypeError): - unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x) + unpackb(b"", object_hook=lambda x: x, object_pairs_hook=lambda x: x) + def test_bad_hook(): with raises(TypeError): - packed = packb([3, 1+2j], default=lambda o: o) + packed = packb([3, 1 + 2j], default=lambda o: o) unpacked = unpackb(packed, use_list=1) + def _arr_to_str(arr): - return ''.join(str(c) for c in arr) + return "".join(str(c) for c in arr) + def test_array_hook(): - packed = packb([1,2,3]) + packed = packb([1, 2, 3]) unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1) - assert unpacked == '123' + assert unpacked == "123" class DecodeError(Exception): pass + def bad_complex_decoder(o): raise DecodeError("Ooops!") def test_an_exception_in_objecthook1(): with raises(DecodeError): - packed = packb({1: {'__complex__': True, 'real': 1, 'imag': 2}}) + packed = packb({1: {"__complex__": True, "real": 1, "imag": 2}}) unpackb(packed, object_hook=bad_complex_decoder) def test_an_exception_in_objecthook2(): with raises(DecodeError): - packed = packb({1: [{'__complex__': True, 'real': 1, 'imag': 2}]}) + packed = packb({1: [{"__complex__": True, "real": 1, "imag": 2}]}) unpackb(packed, list_hook=bad_complex_decoder, use_list=1) diff --git a/test/test_pack.py b/test/test_pack.py index b6752e5..de212ef 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -17,20 +17,46 @@ def check(data, use_list=False): re = unpackb(packb(data), use_list=use_list) assert re == data + def testPack(): test_data = [ - 0, 1, 127, 128, 255, 256, 65535, 65536, 4294967295, 4294967296, - -1, -32, -33, -128, -129, -32768, -32769, -4294967296, -4294967297, - 1.0, - b"", b"a", b"a"*31, b"a"*32, - None, True, False, - (), ((),), ((), None,), + 0, + 1, + 127, + 128, + 255, + 256, + 65535, + 65536, + 4294967295, + 4294967296, + -1, + -32, + -33, + -128, + -129, + -32768, + -32769, + -4294967296, + -4294967297, + 1.0, + b"", + b"a", + b"a" * 31, + b"a" * 32, + None, + True, + False, + (), + ((),), + ((), None,), {None: 0}, - (1<<23), - ] + (1 << 23), + ] for td in test_data: check(td) + def testPackUnicode(): test_data = ["", "abcd", ["defgh"], "РуÑÑкий текÑÑ‚"] for td in test_data: @@ -41,43 +67,64 @@ def testPackUnicode(): re = Unpacker(BytesIO(data), raw=False, use_list=1).unpack() assert re == td + def testPackBytes(): test_data = [ - b"", b"abcd", (b"defgh",), - ] + b"", + b"abcd", + (b"defgh",), + ] for td in test_data: check(td) + def testPackByteArrays(): test_data = [ - bytearray(b""), bytearray(b"abcd"), (bytearray(b"defgh"),), - ] + bytearray(b""), + bytearray(b"abcd"), + (bytearray(b"defgh"),), + ] for td in test_data: check(td) -@pytest.mark.skipif(sys.version_info < (3,0), reason="Python 2 passes invalid surrogates") + +@pytest.mark.skipif( + sys.version_info < (3, 0), reason="Python 2 passes invalid surrogates" +) def testIgnoreUnicodeErrors(): - re = unpackb(packb(b'abc\xeddef', use_bin_type=False), - raw=False, unicode_errors='ignore') + re = unpackb( + packb(b"abc\xeddef", use_bin_type=False), raw=False, unicode_errors="ignore" + ) assert re == "abcdef" + def testStrictUnicodeUnpack(): - packed = packb(b'abc\xeddef', use_bin_type=False) + packed = packb(b"abc\xeddef", use_bin_type=False) with pytest.raises(UnicodeDecodeError): unpackb(packed, raw=False, use_list=1) -@pytest.mark.skipif(sys.version_info < (3,0), reason="Python 2 passes invalid surrogates") + +@pytest.mark.skipif( + sys.version_info < (3, 0), reason="Python 2 passes invalid surrogates" +) def testIgnoreErrorsPack(): - re = unpackb(packb(u"abc\uDC80\uDCFFdef", use_bin_type=True, unicode_errors='ignore'), raw=False, use_list=1) + re = unpackb( + packb("abc\uDC80\uDCFFdef", use_bin_type=True, unicode_errors="ignore"), + raw=False, + use_list=1, + ) assert re == "abcdef" + def testDecodeBinary(): re = unpackb(packb(b"abc"), use_list=1) assert re == b"abc" + def testPackFloat(): - assert packb(1.0, use_single_float=True) == b'\xca' + struct.pack(str('>f'), 1.0) - assert packb(1.0, use_single_float=False) == b'\xcb' + struct.pack(str('>d'), 1.0) + assert packb(1.0, use_single_float=True) == b"\xca" + struct.pack(str(">f"), 1.0) + assert packb(1.0, use_single_float=False) == b"\xcb" + struct.pack(str(">d"), 1.0) + def testArraySize(sizes=[0, 5, 50, 1000]): bio = BytesIO() @@ -92,6 +139,7 @@ def testArraySize(sizes=[0, 5, 50, 1000]): for size in sizes: assert unpacker.unpack() == list(range(size)) + def test_manualreset(sizes=[0, 5, 50, 1000]): packer = Packer(autoreset=False) for size in sizes: @@ -105,7 +153,8 @@ def test_manualreset(sizes=[0, 5, 50, 1000]): assert unpacker.unpack() == list(range(size)) packer.reset() - assert packer.bytes() == b'' + assert packer.bytes() == b"" + def testMapSize(sizes=[0, 5, 50, 1000]): bio = BytesIO() @@ -113,8 +162,8 @@ def testMapSize(sizes=[0, 5, 50, 1000]): for size in sizes: bio.write(packer.pack_map_header(size)) for i in range(size): - bio.write(packer.pack(i)) # key - bio.write(packer.pack(i * 2)) # value + bio.write(packer.pack(i)) # key + bio.write(packer.pack(i * 2)) # value bio.seek(0) unpacker = Unpacker(bio) @@ -123,21 +172,24 @@ def testMapSize(sizes=[0, 5, 50, 1000]): def test_odict(): - seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)] + seq = [(b"one", 1), (b"two", 2), (b"three", 3), (b"four", 4)] od = OrderedDict(seq) assert unpackb(packb(od), use_list=1) == dict(seq) + def pair_hook(seq): return list(seq) + assert unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1) == seq def test_pairlist(): - pairlist = [(b'a', 1), (2, b'b'), (b'foo', b'bar')] + pairlist = [(b"a", 1), (2, b"b"), (b"foo", b"bar")] packer = Packer() packed = packer.pack_map_pairs(pairlist) unpacked = unpackb(packed, object_pairs_hook=list) assert pairlist == unpacked + def test_get_buffer(): packer = Packer(autoreset=0, use_bin_type=True) packer.pack([1, 2]) diff --git a/test/test_read_size.py b/test/test_read_size.py index 4e6c2b9..8d8df64 100644 --- a/test/test_read_size.py +++ b/test/test_read_size.py @@ -1,66 +1,71 @@ """Test Unpacker's read_array_header and read_map_header methods""" from msgpack import packb, Unpacker, OutOfData + UnexpectedTypeException = ValueError + def test_read_array_header(): unpacker = Unpacker() - unpacker.feed(packb(['a', 'b', 'c'])) + unpacker.feed(packb(["a", "b", "c"])) assert unpacker.read_array_header() == 3 - assert unpacker.unpack() == b'a' - assert unpacker.unpack() == b'b' - assert unpacker.unpack() == b'c' + assert unpacker.unpack() == b"a" + assert unpacker.unpack() == b"b" + assert unpacker.unpack() == b"c" try: unpacker.unpack() - assert 0, 'should raise exception' + assert 0, "should raise exception" except OutOfData: - assert 1, 'okay' + assert 1, "okay" def test_read_map_header(): unpacker = Unpacker() - unpacker.feed(packb({'a': 'A'})) + unpacker.feed(packb({"a": "A"})) assert unpacker.read_map_header() == 1 - assert unpacker.unpack() == B'a' - assert unpacker.unpack() == B'A' + assert unpacker.unpack() == b"a" + assert unpacker.unpack() == b"A" try: unpacker.unpack() - assert 0, 'should raise exception' + assert 0, "should raise exception" except OutOfData: - assert 1, 'okay' + assert 1, "okay" + def test_incorrect_type_array(): unpacker = Unpacker() unpacker.feed(packb(1)) try: unpacker.read_array_header() - assert 0, 'should raise exception' + assert 0, "should raise exception" except UnexpectedTypeException: - assert 1, 'okay' + assert 1, "okay" + def test_incorrect_type_map(): unpacker = Unpacker() unpacker.feed(packb(1)) try: unpacker.read_map_header() - assert 0, 'should raise exception' + assert 0, "should raise exception" except UnexpectedTypeException: - assert 1, 'okay' + assert 1, "okay" + def test_correct_type_nested_array(): unpacker = Unpacker() - unpacker.feed(packb({'a': ['b', 'c', 'd']})) + unpacker.feed(packb({"a": ["b", "c", "d"]})) try: unpacker.read_array_header() - assert 0, 'should raise exception' + assert 0, "should raise exception" except UnexpectedTypeException: - assert 1, 'okay' + assert 1, "okay" + def test_incorrect_type_nested_map(): unpacker = Unpacker() - unpacker.feed(packb([{'a': 'b'}])) + unpacker.feed(packb([{"a": "b"}])) try: unpacker.read_map_header() - assert 0, 'should raise exception' + assert 0, "should raise exception" except UnexpectedTypeException: - assert 1, 'okay' - + assert 1, "okay" diff --git a/test/test_seq.py b/test/test_seq.py index fed9ff4..0d5d806 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -7,8 +7,9 @@ import msgpack binarydata = bytes(bytearray(range(256))) + def gen_binary_data(idx): - return binarydata[:idx % 300] + return binarydata[: idx % 300] def test_exceeding_unpacker_read_size(): @@ -18,10 +19,10 @@ def test_exceeding_unpacker_read_size(): NUMBER_OF_STRINGS = 6 read_size = 16 - # 5 ok for read_size=16, while 6 glibc detected *** python: double free or corruption (fasttop): - # 20 ok for read_size=256, while 25 segfaults / glibc detected *** python: double free or corruption (!prev) - # 40 ok for read_size=1024, while 50 introduces errors - # 7000 ok for read_size=1024*1024, while 8000 leads to glibc detected *** python: double free or corruption (!prev): + # 5 ok for read_size=16, while 6 glibc detected *** python: double free or corruption (fasttop): + # 20 ok for read_size=256, while 25 segfaults / glibc detected *** python: double free or corruption (!prev) + # 40 ok for read_size=1024, while 50 introduces errors + # 7000 ok for read_size=1024*1024, while 8000 leads to glibc detected *** python: double free or corruption (!prev): for idx in range(NUMBER_OF_STRINGS): data = gen_binary_data(idx) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 59718f5..e576571 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -10,102 +10,115 @@ from pytest import raises def test_partialdata(): unpacker = Unpacker() - unpacker.feed(b'\xa5') - with raises(StopIteration): next(iter(unpacker)) - unpacker.feed(b'h') - with raises(StopIteration): next(iter(unpacker)) - unpacker.feed(b'a') - with raises(StopIteration): next(iter(unpacker)) - unpacker.feed(b'l') - with raises(StopIteration): next(iter(unpacker)) - unpacker.feed(b'l') - with raises(StopIteration): next(iter(unpacker)) - unpacker.feed(b'o') - assert next(iter(unpacker)) == b'hallo' + unpacker.feed(b"\xa5") + with raises(StopIteration): + next(iter(unpacker)) + unpacker.feed(b"h") + with raises(StopIteration): + next(iter(unpacker)) + unpacker.feed(b"a") + with raises(StopIteration): + next(iter(unpacker)) + unpacker.feed(b"l") + with raises(StopIteration): + next(iter(unpacker)) + unpacker.feed(b"l") + with raises(StopIteration): + next(iter(unpacker)) + unpacker.feed(b"o") + assert next(iter(unpacker)) == b"hallo" + def test_foobar(): unpacker = Unpacker(read_size=3, use_list=1) - unpacker.feed(b'foobar') - assert unpacker.unpack() == ord(b'f') - assert unpacker.unpack() == ord(b'o') - assert unpacker.unpack() == ord(b'o') - assert unpacker.unpack() == ord(b'b') - assert unpacker.unpack() == ord(b'a') - assert unpacker.unpack() == ord(b'r') + unpacker.feed(b"foobar") + assert unpacker.unpack() == ord(b"f") + assert unpacker.unpack() == ord(b"o") + assert unpacker.unpack() == ord(b"o") + assert unpacker.unpack() == ord(b"b") + assert unpacker.unpack() == ord(b"a") + assert unpacker.unpack() == ord(b"r") with raises(OutOfData): unpacker.unpack() - unpacker.feed(b'foo') - unpacker.feed(b'bar') + unpacker.feed(b"foo") + unpacker.feed(b"bar") k = 0 - for o, e in zip(unpacker, 'foobarbaz'): + for o, e in zip(unpacker, "foobarbaz"): assert o == ord(e) k += 1 - assert k == len(b'foobar') + assert k == len(b"foobar") + def test_foobar_skip(): unpacker = Unpacker(read_size=3, use_list=1) - unpacker.feed(b'foobar') - assert unpacker.unpack() == ord(b'f') + unpacker.feed(b"foobar") + assert unpacker.unpack() == ord(b"f") unpacker.skip() - assert unpacker.unpack() == ord(b'o') + assert unpacker.unpack() == ord(b"o") unpacker.skip() - assert unpacker.unpack() == ord(b'a') + assert unpacker.unpack() == ord(b"a") unpacker.skip() with raises(OutOfData): unpacker.unpack() + def test_maxbuffersize(): with raises(ValueError): Unpacker(read_size=5, max_buffer_size=3) unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1) - unpacker.feed(b'fo') + unpacker.feed(b"fo") with raises(BufferFull): - unpacker.feed(b'ob') - unpacker.feed(b'o') - assert ord('f') == next(unpacker) - unpacker.feed(b'b') - assert ord('o') == next(unpacker) - assert ord('o') == next(unpacker) - assert ord('b') == next(unpacker) + unpacker.feed(b"ob") + unpacker.feed(b"o") + assert ord("f") == next(unpacker) + unpacker.feed(b"b") + assert ord("o") == next(unpacker) + assert ord("o") == next(unpacker) + assert ord("b") == next(unpacker) def test_readbytes(): unpacker = Unpacker(read_size=3) - unpacker.feed(b'foobar') - assert unpacker.unpack() == ord(b'f') - assert unpacker.read_bytes(3) == b'oob' - assert unpacker.unpack() == ord(b'a') - assert unpacker.unpack() == ord(b'r') + unpacker.feed(b"foobar") + assert unpacker.unpack() == ord(b"f") + assert unpacker.read_bytes(3) == b"oob" + assert unpacker.unpack() == ord(b"a") + assert unpacker.unpack() == ord(b"r") # Test buffer refill - unpacker = Unpacker(io.BytesIO(b'foobar'), read_size=3) - assert unpacker.unpack() == ord(b'f') - assert unpacker.read_bytes(3) == b'oob' - assert unpacker.unpack() == ord(b'a') - assert unpacker.unpack() == ord(b'r') + unpacker = Unpacker(io.BytesIO(b"foobar"), read_size=3) + assert unpacker.unpack() == ord(b"f") + assert unpacker.read_bytes(3) == b"oob" + assert unpacker.unpack() == ord(b"a") + assert unpacker.unpack() == ord(b"r") + def test_issue124(): unpacker = Unpacker() - unpacker.feed(b'\xa1?\xa1!') - assert tuple(unpacker) == (b'?', b'!') + unpacker.feed(b"\xa1?\xa1!") + assert tuple(unpacker) == (b"?", b"!") assert tuple(unpacker) == () unpacker.feed(b"\xa1?\xa1") - assert tuple(unpacker) == (b'?',) + assert tuple(unpacker) == (b"?",) assert tuple(unpacker) == () unpacker.feed(b"!") - assert tuple(unpacker) == (b'!',) + assert tuple(unpacker) == (b"!",) assert tuple(unpacker) == () def test_unpack_tell(): stream = io.BytesIO() - messages = [2**i-1 for i in range(65)] - messages += [-(2**i) for i in range(1, 64)] - messages += [b'hello', b'hello'*1000, list(range(20)), - {i: bytes(i)*i for i in range(10)}, - {i: bytes(i)*i for i in range(32)}] + messages = [2 ** i - 1 for i in range(65)] + messages += [-(2 ** i) for i in range(1, 64)] + messages += [ + b"hello", + b"hello" * 1000, + list(range(20)), + {i: bytes(i) * i for i in range(10)}, + {i: bytes(i) * i for i in range(32)}, + ] offsets = [] for m in messages: pack(m, stream) diff --git a/test/test_stricttype.py b/test/test_stricttype.py index 87e7c1c..78e1723 100644 --- a/test/test_stricttype.py +++ b/test/test_stricttype.py @@ -5,30 +5,32 @@ from msgpack import packb, unpackb, ExtType def test_namedtuple(): - T = namedtuple('T', "foo bar") + T = namedtuple("T", "foo bar") + def default(o): if isinstance(o, T): return dict(o._asdict()) - raise TypeError('Unsupported type %s' % (type(o),)) + raise TypeError("Unsupported type %s" % (type(o),)) + packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default) unpacked = unpackb(packed, raw=False) - assert unpacked == {'foo': 1, 'bar': 42} + assert unpacked == {"foo": 1, "bar": 42} def test_tuple(): - t = ('one', 2, b'three', (4, )) + t = ("one", 2, b"three", (4,)) def default(o): if isinstance(o, tuple): return { - '__type__': 'tuple', - 'value': list(o), - } - raise TypeError('Unsupported type %s' % (type(o),)) + "__type__": "tuple", + "value": list(o), + } + raise TypeError("Unsupported type %s" % (type(o),)) def convert(o): - if o.get('__type__') == 'tuple': - return tuple(o['value']) + if o.get("__type__") == "tuple": + return tuple(o["value"]) return o data = packb(t, strict_types=True, use_bin_type=True, default=default) @@ -38,7 +40,7 @@ def test_tuple(): def test_tuple_ext(): - t = ('one', 2, b'three', (4, )) + t = ("one", 2, b"three", (4,)) MSGPACK_EXT_TYPE_TUPLE = 0 @@ -46,7 +48,8 @@ def test_tuple_ext(): if isinstance(o, tuple): # Convert to list and pack payload = packb( - list(o), strict_types=True, use_bin_type=True, default=default) + list(o), strict_types=True, use_bin_type=True, default=default + ) return ExtType(MSGPACK_EXT_TYPE_TUPLE, payload) raise TypeError(repr(o)) @@ -54,7 +57,7 @@ def test_tuple_ext(): if code == MSGPACK_EXT_TYPE_TUPLE: # Unpack and convert to tuple return tuple(unpackb(payload, raw=False, ext_hook=convert)) - raise ValueError('Unknown Ext code {}'.format(code)) + raise ValueError("Unknown Ext code {}".format(code)) data = packb(t, strict_types=True, use_bin_type=True, default=default) expected = unpackb(data, raw=False, ext_hook=convert) diff --git a/test/test_subtype.py b/test/test_subtype.py index 6807508..d91d455 100644 --- a/test/test_subtype.py +++ b/test/test_subtype.py @@ -4,16 +4,21 @@ from msgpack import packb, unpackb from collections import namedtuple + class MyList(list): pass + class MyDict(dict): pass + class MyTuple(tuple): pass -MyNamedTuple = namedtuple('MyNamedTuple', 'x y') + +MyNamedTuple = namedtuple("MyNamedTuple", "x y") + def test_types(): assert packb(MyDict()) == packb(dict()) diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 55c2f6d..1348e69 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -4,34 +4,34 @@ from msgpack import Timestamp def test_timestamp(): # timestamp32 - ts = Timestamp(2**32 - 1) + ts = Timestamp(2 ** 32 - 1) assert ts.to_bytes() == b"\xff\xff\xff\xff" packed = msgpack.packb(ts) assert packed == b"\xd6\xff" + ts.to_bytes() unpacked = msgpack.unpackb(packed) assert ts == unpacked - assert ts.seconds == 2**32 - 1 and ts.nanoseconds == 0 + assert ts.seconds == 2 ** 32 - 1 and ts.nanoseconds == 0 # timestamp64 - ts = Timestamp(2**34 - 1, 999999999) + ts = Timestamp(2 ** 34 - 1, 999999999) assert ts.to_bytes() == b"\xee\x6b\x27\xff\xff\xff\xff\xff" packed = msgpack.packb(ts) assert packed == b"\xd7\xff" + ts.to_bytes() unpacked = msgpack.unpackb(packed) assert ts == unpacked - assert ts.seconds == 2**34 - 1 and ts.nanoseconds == 999999999 + assert ts.seconds == 2 ** 34 - 1 and ts.nanoseconds == 999999999 # timestamp96 - ts = Timestamp(2**63 - 1, 999999999) + ts = Timestamp(2 ** 63 - 1, 999999999) assert ts.to_bytes() == b"\x3b\x9a\xc9\xff\x7f\xff\xff\xff\xff\xff\xff\xff" packed = msgpack.packb(ts) assert packed == b"\xc7\x0c\xff" + ts.to_bytes() unpacked = msgpack.unpackb(packed) assert ts == unpacked - assert ts.seconds == 2**63 - 1 and ts.nanoseconds == 999999999 + assert ts.seconds == 2 ** 63 - 1 and ts.nanoseconds == 999999999 # negative fractional - ts = Timestamp(-2.3) #s: -3, ns: 700000000 + ts = Timestamp(-2.3) # s: -3, ns: 700000000 assert ts.to_bytes() == b"\x29\xb9\x27\x00\xff\xff\xff\xff\xff\xff\xff\xfd" packed = msgpack.packb(ts) assert packed == b"\xc7\x0c\xff" + ts.to_bytes() diff --git a/test/test_unpack.py b/test/test_unpack.py index 00a1061..bc74c4d 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -5,7 +5,7 @@ from pytest import raises, mark def test_unpack_array_header_from_file(): - f = BytesIO(packb([1,2,3,4])) + f = BytesIO(packb([1, 2, 3, 4])) unpacker = Unpacker(f) assert unpacker.read_array_header() == 4 assert unpacker.unpack() == 1 @@ -16,8 +16,10 @@ def test_unpack_array_header_from_file(): unpacker.unpack() -@mark.skipif("not hasattr(sys, 'getrefcount') == True", - reason='sys.getrefcount() is needed to pass this test') +@mark.skipif( + "not hasattr(sys, 'getrefcount') == True", + reason="sys.getrefcount() is needed to pass this test", +) def test_unpacker_hook_refcnt(): result = [] @@ -43,12 +45,9 @@ def test_unpacker_hook_refcnt(): def test_unpacker_ext_hook(): - class MyUnpacker(Unpacker): - def __init__(self): - super(MyUnpacker, self).__init__( - ext_hook=self._hook, raw=False) + super(MyUnpacker, self).__init__(ext_hook=self._hook, raw=False) def _hook(self, code, data): if code == 1: @@ -57,15 +56,15 @@ def test_unpacker_ext_hook(): return ExtType(code, data) unpacker = MyUnpacker() - unpacker.feed(packb({'a': 1})) - assert unpacker.unpack() == {'a': 1} - unpacker.feed(packb({'a': ExtType(1, b'123')})) - assert unpacker.unpack() == {'a': 123} - unpacker.feed(packb({'a': ExtType(2, b'321')})) - assert unpacker.unpack() == {'a': ExtType(2, b'321')} + unpacker.feed(packb({"a": 1})) + assert unpacker.unpack() == {"a": 1} + unpacker.feed(packb({"a": ExtType(1, b"123")})) + assert unpacker.unpack() == {"a": 123} + unpacker.feed(packb({"a": ExtType(2, b"321")})) + assert unpacker.unpack() == {"a": ExtType(2, b"321")} -if __name__ == '__main__': +if __name__ == "__main__": test_unpack_array_header_from_file() test_unpacker_hook_refcnt() test_unpacker_ext_hook() From bc8c86203af8d36152c9c72ea22e895db2ed3fe0 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 5 Dec 2019 18:53:49 +0900 Subject: [PATCH 1010/1172] blacken all files. --- benchmark/benchmark.py | 20 +++-- docs/conf.py | 164 ++++++++++++++++++++--------------------- setup.py | 119 ++++++++++++++++-------------- 3 files changed, 159 insertions(+), 144 deletions(-) diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 80819c6..82d0ddb 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -1,6 +1,8 @@ from msgpack import fallback + try: from msgpack import _unpacker, _packer + has_ext = True except ImportError: has_ext = False @@ -9,7 +11,7 @@ import timeit def profile(name, func): times = timeit.repeat(func, number=1000, repeat=4) - times = ', '.join(["%8f" % t for t in times]) + times = ", ".join(["%8f" % t for t in times]) print("%-30s %40s" % (name, times)) @@ -18,17 +20,19 @@ def simple(name, data): packer = _packer.Packer() profile("packing %s (ext)" % name, lambda: packer.pack(data)) packer = fallback.Packer() - profile('packing %s (fallback)' % name, lambda: packer.pack(data)) + profile("packing %s (fallback)" % name, lambda: packer.pack(data)) data = packer.pack(data) if has_ext: - profile('unpacking %s (ext)' % name, lambda: _unpacker.unpackb(data)) - profile('unpacking %s (fallback)' % name, lambda: fallback.unpackb(data)) + profile("unpacking %s (ext)" % name, lambda: _unpacker.unpackb(data)) + profile("unpacking %s (fallback)" % name, lambda: fallback.unpackb(data)) + def main(): - simple("integers", [7]*10000) - simple("bytes", [b'x'*n for n in range(100)]*10) - simple("lists", [[]]*10000) - simple("dicts", [{}]*10000) + simple("integers", [7] * 10000) + simple("bytes", [b"x" * n for n in range(100)] * 10) + simple("lists", [[]] * 10000) + simple("dicts", [{}] * 10000) + main() diff --git a/docs/conf.py b/docs/conf.py index 47d745a..36fa76e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,32 +16,32 @@ import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] +extensions = ["sphinx.ext.autodoc", "sphinx.ext.viewcode"] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'msgpack' -copyright = u'2013, INADA Naoki' +project = u"msgpack" +copyright = u"2013, INADA Naoki" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -49,176 +49,170 @@ copyright = u'2013, INADA Naoki' # # The short X.Y version. # The full version, including alpha/beta/rc tags. -version = release = '0.5' +version = release = "0.5" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' today_fmt = "%Y-%m-%d" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinxdoc' +html_theme = "sphinxdoc" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'msgpackdoc' +htmlhelp_basename = "msgpackdoc" # -- Options for LaTeX output -------------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'msgpack.tex', u'msgpack Documentation', - u'Author', 'manual'), + ("index", "msgpack.tex", u"msgpack Documentation", u"Author", "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'msgpack', u'msgpack Documentation', - [u'Author'], 1) -] +man_pages = [("index", "msgpack", u"msgpack Documentation", [u"Author"], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ @@ -227,59 +221,65 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'msgpack', u'msgpack Documentation', - u'Author', 'msgpack', 'One line description of project.', - 'Miscellaneous'), + ( + "index", + "msgpack", + u"msgpack Documentation", + u"Author", + "msgpack", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # -- Options for Epub output --------------------------------------------------- # Bibliographic Dublin Core info. -epub_title = u'msgpack' -epub_author = u'Author' -epub_publisher = u'Author' -epub_copyright = u'2013, Author' +epub_title = u"msgpack" +epub_author = u"Author" +epub_publisher = u"Author" +epub_copyright = u"2013, Author" # The language of the text. It defaults to the language option # or en if the language is not set. -#epub_language = '' +# epub_language = '' # The scheme of the identifier. Typical schemes are ISBN or URL. -#epub_scheme = '' +# epub_scheme = '' # The unique identifier of the text. This can be a ISBN number # or the project homepage. -#epub_identifier = '' +# epub_identifier = '' # A unique identification for the text. -#epub_uid = '' +# epub_uid = '' # A tuple containing the cover image and cover page html template filenames. -#epub_cover = () +# epub_cover = () # HTML files that should be inserted before the pages created by sphinx. # The format is a list of tuples containing the path and title. -#epub_pre_files = [] +# epub_pre_files = [] # HTML files shat should be inserted after the pages created by sphinx. # The format is a list of tuples containing the path and title. -#epub_post_files = [] +# epub_post_files = [] # A list of files that should not be packed into the epub file. -#epub_exclude_files = [] +# epub_exclude_files = [] # The depth of the table of contents in toc.ncx. -#epub_tocdepth = 3 +# epub_tocdepth = 3 # Allow duplicate toc entries. -#epub_tocdup = True +# epub_tocdup = True diff --git a/setup.py b/setup.py index 77b81c6..a8c2306 100755 --- a/setup.py +++ b/setup.py @@ -17,11 +17,14 @@ PY2 = sys.version_info[0] == 2 # for building transitional package. TRANSITIONAL = False + class NoCython(Exception): pass + try: import Cython.Compiler.Main as cython_compiler + have_cython = True except ImportError: have_cython = False @@ -31,16 +34,19 @@ def cythonize(src): sys.stderr.write("cythonize: %r\n" % (src,)) cython_compiler.compile([src], cplus=True) + def ensure_source(src): - pyx = os.path.splitext(src)[0] + '.pyx' + pyx = os.path.splitext(src)[0] + ".pyx" if not os.path.exists(src): if not have_cython: raise NoCython cythonize(pyx) - elif (os.path.exists(pyx) and - os.stat(src).st_mtime < os.stat(pyx).st_mtime and - have_cython): + elif ( + os.path.exists(pyx) + and os.stat(src).st_mtime < os.stat(pyx).st_mtime + and have_cython + ): cythonize(pyx) return src @@ -63,77 +69,82 @@ class BuildExt(build_ext): print(e) -exec(open('msgpack/_version.py').read()) +exec(open("msgpack/_version.py").read()) -version_str = '.'.join(str(x) for x in version[:3]) -if len(version) > 3 and version[3] != 'final': +version_str = ".".join(str(x) for x in version[:3]) +if len(version) > 3 and version[3] != "final": version_str += version[3] # Cython is required for sdist class Sdist(sdist): def __init__(self, *args, **kwargs): - cythonize('msgpack/_cmsgpack.pyx') + cythonize("msgpack/_cmsgpack.pyx") sdist.__init__(self, *args, **kwargs) -libraries = [] -if sys.platform == 'win32': - libraries.append('ws2_32') -if sys.byteorder == 'big': - macros = [('__BIG_ENDIAN__', '1')] +libraries = [] +if sys.platform == "win32": + libraries.append("ws2_32") + +if sys.byteorder == "big": + macros = [("__BIG_ENDIAN__", "1")] else: - macros = [('__LITTLE_ENDIAN__', '1')] + macros = [("__LITTLE_ENDIAN__", "1")] ext_modules = [] if not PYPY and not PY2: - ext_modules.append(Extension('msgpack._cmsgpack', - sources=['msgpack/_cmsgpack.cpp'], - libraries=libraries, - include_dirs=['.'], - define_macros=macros, - )) + ext_modules.append( + Extension( + "msgpack._cmsgpack", + sources=["msgpack/_cmsgpack.cpp"], + libraries=libraries, + include_dirs=["."], + define_macros=macros, + ) + ) del libraries, macros -desc = 'MessagePack (de)serializer.' -with io.open('README.rst', encoding='utf-8') as f: +desc = "MessagePack (de)serializer." +with io.open("README.rst", encoding="utf-8") as f: long_desc = f.read() del f -name = 'msgpack' +name = "msgpack" if TRANSITIONAL: - name = 'msgpack-python' + name = "msgpack-python" long_desc = "This package is deprecated. Install msgpack instead." -setup(name=name, - author='INADA Naoki', - author_email='songofacandy@gmail.com', - version=version_str, - cmdclass={'build_ext': BuildExt, 'sdist': Sdist}, - ext_modules=ext_modules, - packages=['msgpack'], - description=desc, - long_description=long_desc, - long_description_content_type="text/x-rst", - url='https://msgpack.org/', - project_urls = { - 'Documentation': 'https://msgpack-python.readthedocs.io/', - 'Source': 'https://github.com/msgpack/msgpack-python', - 'Tracker': 'https://github.com/msgpack/msgpack-python/issues', - }, - license='Apache 2.0', - classifiers=[ - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: Apache Software License', - ], +setup( + name=name, + author="INADA Naoki", + author_email="songofacandy@gmail.com", + version=version_str, + cmdclass={"build_ext": BuildExt, "sdist": Sdist}, + ext_modules=ext_modules, + packages=["msgpack"], + description=desc, + long_description=long_desc, + long_description_content_type="text/x-rst", + url="https://msgpack.org/", + project_urls={ + "Documentation": "https://msgpack-python.readthedocs.io/", + "Source": "https://github.com/msgpack/msgpack-python", + "Tracker": "https://github.com/msgpack/msgpack-python/issues", + }, + license="Apache 2.0", + classifiers=[ + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + ], ) From af4eea430e2f176f17fff5abe781dd83f55d4657 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 5 Dec 2019 18:54:14 +0900 Subject: [PATCH 1011/1172] travis: Add Black --- .travis.yml | 9 ++++++++- Makefile | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7b298af..9d3ae74 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,15 @@ python: matrix: include: + - name: Black + language: python + python: 3.8 + install: + - pip install black + script: + - black --check --diff . + - name: 32bit build - sudo: required language: python services: - docker diff --git a/Makefile b/Makefile index 5828ed4..a1edc88 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,10 @@ all: cython python setup.py build_ext -i -f +.PHONY: black +black: + black . + .PHONY: cython cython: cython --cplus msgpack/_cmsgpack.pyx From 9ae43709e42092c7f6a4e990d696d9005fa1623d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 5 Dec 2019 20:20:53 +0900 Subject: [PATCH 1012/1172] Drop old buffer protocol support (#383) --- msgpack/_unpacker.pyx | 60 ++++++++++++++++--------------------------- 1 file changed, 22 insertions(+), 38 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 6dedd30..3c9b7b3 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -109,38 +109,26 @@ def default_read_extended_type(typecode, data): cdef inline int get_data_from_buffer(object obj, Py_buffer *view, char **buf, - Py_ssize_t *buffer_len, - int *new_protocol) except 0: + Py_ssize_t *buffer_len) except 0: cdef object contiguous cdef Py_buffer tmp - if PyObject_CheckBuffer(obj): - new_protocol[0] = 1 - if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1: - raise - if view.itemsize != 1: - PyBuffer_Release(view) - raise BufferError("cannot unpack from multi-byte object") - if PyBuffer_IsContiguous(view, b'A') == 0: - PyBuffer_Release(view) - # create a contiguous copy and get buffer - contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, b'C') - PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE) - # view must hold the only reference to contiguous, - # so memory is freed when view is released - Py_DECREF(contiguous) - buffer_len[0] = view.len - buf[0] = view.buf - return 1 - else: - new_protocol[0] = 0 - if PyObject_AsReadBuffer(obj, buf, buffer_len) == -1: - raise BufferError("could not get memoryview") - PyErr_WarnEx(RuntimeWarning, - "using old buffer interface to unpack %s; " - "this leads to unpacking errors if slicing is used and " - "will be removed in a future version" % type(obj), - 1) - return 1 + if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1: + raise + if view.itemsize != 1: + PyBuffer_Release(view) + raise BufferError("cannot unpack from multi-byte object") + if PyBuffer_IsContiguous(view, b'A') == 0: + PyBuffer_Release(view) + # create a contiguous copy and get buffer + contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, b'C') + PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE) + # view must hold the only reference to contiguous, + # so memory is freed when view is released + Py_DECREF(contiguous) + buffer_len[0] = view.len + buf[0] = view.buf + return 1 + def unpackb(object packed, *, object object_hook=None, object list_hook=None, bint use_list=True, bint raw=True, bint strict_map_key=False, @@ -172,12 +160,11 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, cdef char* buf = NULL cdef Py_ssize_t buf_len cdef const char* cerr = NULL - cdef int new_protocol = 0 if unicode_errors is not None: cerr = unicode_errors - get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol) + get_data_from_buffer(packed, &view, &buf, &buf_len) if max_str_len == -1: max_str_len = buf_len @@ -196,8 +183,7 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) ret = unpack_construct(&ctx, buf, buf_len, &off) finally: - if new_protocol: - PyBuffer_Release(&view); + PyBuffer_Release(&view); if ret == 1: obj = unpack_data(&ctx) @@ -392,7 +378,6 @@ cdef class Unpacker(object): def feed(self, object next_bytes): """Append `next_bytes` to internal buffer.""" cdef Py_buffer pybuff - cdef int new_protocol = 0 cdef char* buf cdef Py_ssize_t buf_len @@ -400,12 +385,11 @@ cdef class Unpacker(object): raise AssertionError( "unpacker.feed() is not be able to use with `file_like`.") - get_data_from_buffer(next_bytes, &pybuff, &buf, &buf_len, &new_protocol) + get_data_from_buffer(next_bytes, &pybuff, &buf, &buf_len) try: self.append_buffer(buf, buf_len) finally: - if new_protocol: - PyBuffer_Release(&pybuff) + PyBuffer_Release(&pybuff) cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len): cdef: From 9f4b2d53b77c5ccd96e3ceb359747960cbf03bd4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 5 Dec 2019 20:47:01 +0900 Subject: [PATCH 1013/1172] Remove deprecated submodule unpack (#385) --- msgpack/_unpacker.pyx | 8 -------- msgpack/fallback.py | 10 ---------- 2 files changed, 18 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 3c9b7b3..3a9d494 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -200,14 +200,6 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, raise ValueError("Unpack failed: error = %d" % (ret,)) -def unpack(object stream, **kwargs): - PyErr_WarnEx( - DeprecationWarning, - "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", 1) - data = stream.read() - return unpackb(data, **kwargs) - - cdef class Unpacker(object): """Streaming unpacker. diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 577e571..3faacbf 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -131,16 +131,6 @@ def _get_data_from_buffer(obj): return view -def unpack(stream, **kwargs): - warnings.warn( - "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", - DeprecationWarning, - stacklevel=2, - ) - data = stream.read() - return unpackb(data, **kwargs) - - def unpackb(packed, **kwargs): """ Unpack an object from `packed`. From de320488ae494b85a03b60dd33f91b650033d775 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 5 Dec 2019 20:47:20 +0900 Subject: [PATCH 1014/1172] fallback: Remove old buffer protocol support (#384) --- msgpack/fallback.py | 16 +--------------- test/test_buffer.py | 8 ++++---- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 3faacbf..9de3553 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -111,21 +111,7 @@ def _check_type_strict(obj, t, type=type, tuple=tuple): def _get_data_from_buffer(obj): - try: - view = memoryview(obj) - except TypeError: - # try to use legacy buffer protocol if 2.7, otherwise re-raise - if PY2: - view = memoryview(buffer(obj)) - warnings.warn( - "using old buffer interface to unpack %s; " - "this leads to unpacking errors if slicing is used and " - "will be removed in a future version" % type(obj), - RuntimeWarning, - stacklevel=3, - ) - else: - raise + view = memoryview(obj) if view.itemsize != 1: raise ValueError("cannot unpack from multi-byte object") return view diff --git a/test/test_buffer.py b/test/test_buffer.py index 64fbdef..da68b27 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -1,17 +1,17 @@ #!/usr/bin/env python # coding: utf-8 +import sys +import pytest from msgpack import packb, unpackb +@pytest.mark.skipif(sys.version_info[0] == 2, reason="Python 2 is not supported") def test_unpack_buffer(): from array import array buf = array("b") - try: - buf.frombytes(packb((b"foo", b"bar"))) - except AttributeError: # PY2 - buf.fromstring(packb((b"foo", b"bar"))) + buf.frombytes(packb((b"foo", b"bar"))) obj = unpackb(buf, use_list=1) assert [b"foo", b"bar"] == obj From 7e9905bdfaecde83ddb1a4575e734a10b055fde9 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 5 Dec 2019 21:34:10 +0900 Subject: [PATCH 1015/1172] Use new msgpack spec by default. (#386) --- README.rst | 70 ++++++++++++----------------------------- msgpack/_packer.pyx | 6 ++-- msgpack/_unpacker.pyx | 16 ++++------ msgpack/fallback.py | 20 +++++------- test/test_buffer.py | 4 +-- test/test_case.py | 11 +++---- test/test_format.py | 10 ++++-- test/test_memoryview.py | 39 +++++++---------------- test/test_newspec.py | 6 ++-- test/test_read_size.py | 10 +++--- test/test_sequnpack.py | 9 +++--- 11 files changed, 75 insertions(+), 126 deletions(-) diff --git a/README.rst b/README.rst index 82b6c02..f9f074f 100644 --- a/README.rst +++ b/README.rst @@ -37,36 +37,16 @@ Sadly, this doesn't work for upgrade install. After `pip install -U msgpack-pyt msgpack is removed and `import msgpack` fail. -Deprecating encoding option -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Compatibility with old format +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -encoding and unicode_errors options are deprecated. +You can use ``use_bin_type=False`` option to pack ``bytes`` +object into raw type in old msgpack spec, instead of bin type in new msgpack spec. -In case of packer, use UTF-8 always. Storing other than UTF-8 is not recommended. +You can unpack old msgpack formatk using ``raw=True`` option. +It unpacks str (raw) type in msgpack into Python bytes. -For backward compatibility, you can use ``use_bin_type=False`` and pack ``bytes`` -object into msgpack raw type. - -In case of unpacker, there is new ``raw`` option. It is ``True`` by default -for backward compatibility, but it is changed to ``False`` in near future. -You can use ``raw=False`` instead of ``encoding='utf-8'``. - -Planned backward incompatible changes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When msgpack 1.0, I planning these breaking changes: - -* packer and unpacker: Remove ``encoding`` and ``unicode_errors`` option. -* packer: Change default of ``use_bin_type`` option from False to True. -* unpacker: Change default of ``raw`` option from True to False. -* unpacker: Reduce all ``max_xxx_len`` options for typical usage. -* unpacker: Remove ``write_bytes`` option from all methods. - -To avoid these breaking changes breaks your application, please: - -* Don't use deprecated options. -* Pass ``use_bin_type`` and ``raw`` options explicitly. -* If your application handle large (>1MB) data, specify ``max_xxx_len`` options too. +See note in below for detail. Install @@ -76,6 +56,7 @@ Install $ pip install msgpack + Pure Python implementation ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -100,6 +81,13 @@ Without extension, using pure Python implementation on CPython runs slowly. How to use ---------- +.. note:: + + In examples below, I use ``raw=False`` and ``use_bin_type=True`` for users + using msgpack < 1.0. + These options are default from msgpack 1.0 so you can omit them. + + One-shot pack & unpack ^^^^^^^^^^^^^^^^^^^^^^ @@ -252,36 +240,18 @@ Notes string and binary type ^^^^^^^^^^^^^^^^^^^^^^ -Early versions of msgpack didn't distinguish string and binary types (like Python 1). +Early versions of msgpack didn't distinguish string and binary types. The type for representing both string and binary types was named **raw**. -For backward compatibility reasons, msgpack-python will still default all -strings to byte strings, unless you specify the ``use_bin_type=True`` option in -the packer. If you do so, it will use a non-standard type called **bin** to -serialize byte arrays, and **raw** becomes to mean **str**. If you want to -distinguish **bin** and **raw** in the unpacker, specify ``raw=False``. - -Note that Python 2 defaults to byte-arrays over Unicode strings: +You can pack into and unpack from this old spec using ``use_bin_type=False`` +and ``raw=True`` options. .. code-block:: pycon >>> import msgpack - >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'])) - ['spam', 'eggs'] - >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), - raw=False) - ['spam', u'eggs'] - -This is the same code in Python 3 (same behaviour, but Python 3 has a -different default): - -.. code-block:: pycon - - >>> import msgpack - >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'])) + >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=False), raw=True) [b'spam', b'eggs'] - >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), - raw=False) + >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), raw=False) [b'spam', 'eggs'] diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index f3bde3f..8cf3c05 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -80,9 +80,7 @@ cdef class Packer(object): :param bool use_bin_type: Use bin type introduced in msgpack spec 2.0 for bytes. - It also enables str8 type for unicode. - Current default value is false, but it will be changed to true - in future version. You should specify it explicitly. + It also enables str8 type for unicode. (default: True) :param bool strict_types: If set to true, types will be checked to be exact. Derived classes @@ -113,7 +111,7 @@ cdef class Packer(object): self.pk.length = 0 def __init__(self, *, default=None, unicode_errors=None, - bint use_single_float=False, bint autoreset=True, bint use_bin_type=False, + bint use_single_float=False, bint autoreset=True, bint use_bin_type=True, bint strict_types=False): self.use_float = use_single_float self.strict_types = strict_types diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 3a9d494..f10e99d 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -131,7 +131,7 @@ cdef inline int get_data_from_buffer(object obj, def unpackb(object packed, *, object object_hook=None, object list_hook=None, - bint use_list=True, bint raw=True, bint strict_map_key=False, + bint use_list=True, bint raw=False, bint strict_map_key=False, unicode_errors=None, object_pairs_hook=None, ext_hook=ExtType, Py_ssize_t max_str_len=-1, @@ -217,12 +217,8 @@ cdef class Unpacker(object): Otherwise, unpack to Python tuple. (default: True) :param bool raw: - If true, unpack msgpack raw to Python bytes (default). - Otherwise, unpack to Python str (or unicode on Python 2) by decoding - with UTF-8 encoding (recommended). - Currently, the default is true, but it will be changed to false in - near future. So you must specify it explicitly for keeping backward - compatibility. + If true, unpack msgpack raw to Python bytes. + Otherwise, unpack to Python str by decoding with UTF-8 encoding (default). :param bool strict_map_key: If true, only str or bytes are accepted for map (dict) keys. @@ -268,13 +264,13 @@ cdef class Unpacker(object): Example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like, raw=False, max_buffer_size=10*1024*1024) + unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024) for o in unpacker: process(o) Example of streaming deserialize from socket:: - unpacker = Unpacker(raw=False, max_buffer_size=10*1024*1024) + unpacker = Unpacker(max_buffer_size=10*1024*1024) while True: buf = sock.recv(1024**2) if not buf: @@ -309,7 +305,7 @@ cdef class Unpacker(object): self.buf = NULL def __init__(self, file_like=None, *, Py_ssize_t read_size=0, - bint use_list=True, bint raw=True, bint strict_map_key=False, + bint use_list=True, bint raw=False, bint strict_map_key=False, object object_hook=None, object object_pairs_hook=None, object list_hook=None, unicode_errors=None, Py_ssize_t max_buffer_size=0, object ext_hook=ExtType, diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 9de3553..fa2f3a8 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -158,7 +158,7 @@ else: class Unpacker(object): """Streaming unpacker. - arguments: + Arguments: :param file_like: File-like object having `.read(n)` method. @@ -172,12 +172,8 @@ class Unpacker(object): Otherwise, unpack to Python tuple. (default: True) :param bool raw: - If true, unpack msgpack raw to Python bytes (default). - Otherwise, unpack to Python str (or unicode on Python 2) by decoding - with UTF-8 encoding (recommended). - Currently, the default is true, but it will be changed to false in - near future. So you must specify it explicitly for keeping backward - compatibility. + If true, unpack msgpack raw to Python bytes. + Otherwise, unpack to Python str by decoding with UTF-8 encoding (default). :param bool strict_map_key: If true, only str or bytes are accepted for map (dict) keys. @@ -226,13 +222,13 @@ class Unpacker(object): Example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like, raw=False, max_buffer_size=10*1024*1024) + unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024) for o in unpacker: process(o) Example of streaming deserialize from socket:: - unpacker = Unpacker(raw=False, max_buffer_size=10*1024*1024) + unpacker = Unpacker(max_buffer_size=10*1024*1024) while True: buf = sock.recv(1024**2) if not buf: @@ -253,7 +249,7 @@ class Unpacker(object): file_like=None, read_size=0, use_list=True, - raw=True, + raw=False, strict_map_key=False, object_hook=None, object_pairs_hook=None, @@ -748,7 +744,7 @@ class Packer(object): :param bool use_bin_type: Use bin type introduced in msgpack spec 2.0 for bytes. - It also enables str8 type for unicode. + It also enables str8 type for unicode. (default: True) :param bool strict_types: If set to true, types will be checked to be exact. Derived classes @@ -769,7 +765,7 @@ class Packer(object): unicode_errors=None, use_single_float=False, autoreset=True, - use_bin_type=False, + use_bin_type=True, strict_types=False, ): self._strict_types = strict_types diff --git a/test/test_buffer.py b/test/test_buffer.py index da68b27..62507cf 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -17,7 +17,7 @@ def test_unpack_buffer(): def test_unpack_bytearray(): - buf = bytearray(packb(("foo", "bar"))) + buf = bytearray(packb((b"foo", b"bar"))) obj = unpackb(buf, use_list=1) assert [b"foo", b"bar"] == obj expected_type = bytes @@ -25,7 +25,7 @@ def test_unpack_bytearray(): def test_unpack_memoryview(): - buf = bytearray(packb(("foo", "bar"))) + buf = bytearray(packb((b"foo", b"bar"))) view = memoryview(buf) obj = unpackb(view, use_list=1) assert [b"foo", b"bar"] == obj diff --git a/test/test_case.py b/test/test_case.py index 3bc1b26..3e60e59 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -1,13 +1,12 @@ #!/usr/bin/env python # coding: utf-8 - from msgpack import packb, unpackb -def check(length, obj): - v = packb(obj) +def check(length, obj, use_bin_type=True): + v = packb(obj, use_bin_type=use_bin_type) assert len(v) == length, "%r length should be %r but get %r" % (obj, length, len(v)) - assert unpackb(v, use_list=0) == obj + assert unpackb(v, use_list=0, raw=not use_bin_type) == obj def test_1(): @@ -56,7 +55,7 @@ def test_9(): def check_raw(overhead, num): - check(num + overhead, b" " * num) + check(num + overhead, b" " * num, use_bin_type=False) def test_fixraw(): @@ -135,4 +134,4 @@ def test_match(): def test_unicode(): - assert unpackb(packb("foobar"), use_list=1) == b"foobar" + assert unpackb(packb(u"foobar"), use_list=1) == u"foobar" diff --git a/test/test_format.py b/test/test_format.py index c2cdfbd..8c2f03f 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -4,8 +4,8 @@ from msgpack import unpackb -def check(src, should, use_list=0): - assert unpackb(src, use_list=use_list) == should +def check(src, should, use_list=0, raw=True): + assert unpackb(src, use_list=use_list, raw=raw) == should def testSimpleValue(): @@ -59,6 +59,12 @@ def testRaw(): b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", (b"", b"a", b"ab", b"", b"a", b"ab"), ) + check( + b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" + b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", + ("", "a", "ab", "", "a", "ab"), + raw=False, + ) def testArray(): diff --git a/test/test_memoryview.py b/test/test_memoryview.py index e1b63b8..86b2c1f 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -1,50 +1,33 @@ #!/usr/bin/env python # coding: utf-8 +import pytest from array import array from msgpack import packb, unpackb import sys -# For Python < 3: -# - array type only supports old buffer interface -# - array.frombytes is not available, must use deprecated array.fromstring -if sys.version_info[0] < 3: - - def make_memoryview(obj): - return memoryview(buffer(obj)) - - def make_array(f, data): - a = array(f) - a.fromstring(data) - return a - - def get_data(a): - return a.tostring() +pytestmark = pytest.mark.skipif( + sys.version_info[0] < 3, reason="Only Python 3 supports buffer protocol" +) -else: - make_memoryview = memoryview - - def make_array(f, data): - a = array(f) - a.frombytes(data) - return a - - def get_data(a): - return a.tobytes() +def make_array(f, data): + a = array(f) + a.frombytes(data) + return a def _runtest(format, nbytes, expected_header, expected_prefix, use_bin_type): # create a new array original_array = array(format) original_array.fromlist([255] * (nbytes // original_array.itemsize)) - original_data = get_data(original_array) - view = make_memoryview(original_array) + original_data = original_array.tobytes() + view = memoryview(original_array) # pack, unpack, and reconstruct array packed = packb(view, use_bin_type=use_bin_type) - unpacked = unpackb(packed) + unpacked = unpackb(packed, raw=(not use_bin_type)) reconstructed_array = make_array(format, unpacked) # check that we got the right amount of data diff --git a/test/test_newspec.py b/test/test_newspec.py index f4f2a23..b7da486 100644 --- a/test/test_newspec.py +++ b/test/test_newspec.py @@ -10,14 +10,16 @@ def test_str8(): assert len(b) == len(data) + 2 assert b[0:2] == header + b"\x20" assert b[2:] == data - assert unpackb(b) == data + assert unpackb(b, raw=True) == data + assert unpackb(b, raw=False) == data.decode() data = b"x" * 255 b = packb(data.decode(), use_bin_type=True) assert len(b) == len(data) + 2 assert b[0:2] == header + b"\xff" assert b[2:] == data - assert unpackb(b) == data + assert unpackb(b, raw=True) == data + assert unpackb(b, raw=False) == data.decode() def test_bin8(): diff --git a/test/test_read_size.py b/test/test_read_size.py index 8d8df64..33a7e7d 100644 --- a/test/test_read_size.py +++ b/test/test_read_size.py @@ -8,9 +8,9 @@ def test_read_array_header(): unpacker = Unpacker() unpacker.feed(packb(["a", "b", "c"])) assert unpacker.read_array_header() == 3 - assert unpacker.unpack() == b"a" - assert unpacker.unpack() == b"b" - assert unpacker.unpack() == b"c" + assert unpacker.unpack() == "a" + assert unpacker.unpack() == "b" + assert unpacker.unpack() == "c" try: unpacker.unpack() assert 0, "should raise exception" @@ -22,8 +22,8 @@ def test_read_map_header(): unpacker = Unpacker() unpacker.feed(packb({"a": "A"})) assert unpacker.read_map_header() == 1 - assert unpacker.unpack() == b"a" - assert unpacker.unpack() == b"A" + assert unpacker.unpack() == "a" + assert unpacker.unpack() == "A" try: unpacker.unpack() assert 0, "should raise exception" diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index e576571..9b69479 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,6 +1,5 @@ #!/usr/bin/env python # coding: utf-8 - import io from msgpack import Unpacker, BufferFull from msgpack import pack @@ -26,7 +25,7 @@ def test_partialdata(): with raises(StopIteration): next(iter(unpacker)) unpacker.feed(b"o") - assert next(iter(unpacker)) == b"hallo" + assert next(iter(unpacker)) == "hallo" def test_foobar(): @@ -98,13 +97,13 @@ def test_readbytes(): def test_issue124(): unpacker = Unpacker() unpacker.feed(b"\xa1?\xa1!") - assert tuple(unpacker) == (b"?", b"!") + assert tuple(unpacker) == ("?", "!") assert tuple(unpacker) == () unpacker.feed(b"\xa1?\xa1") - assert tuple(unpacker) == (b"?",) + assert tuple(unpacker) == ("?",) assert tuple(unpacker) == () unpacker.feed(b"!") - assert tuple(unpacker) == (b"!",) + assert tuple(unpacker) == ("!",) assert tuple(unpacker) == () From 235c6036eabd89d5a28244091ac1e4f091b49679 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Dec 2019 19:28:23 +0900 Subject: [PATCH 1016/1172] travis: Use codecov (#387) --- .travis.yml | 52 +++++++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9d3ae74..378bc80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,11 +12,20 @@ python: - "3.7" - "3.8-dev" + +_pure: &pure + install: + - pip install -U pip + - pip install -U pytest pytest-cov codecov + - pip install . + script: + - pytest --cov=msgpack -v test + matrix: include: - name: Black language: python - python: 3.8 + python: 3.7 install: - pip install black script: @@ -35,38 +44,35 @@ matrix: - docker pull $DOCKER_IMAGE script: - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh - - name: "pypy2.7" - python: "pypy2.7-7.1.1" - install: - - pip install -e . - script: - - py.test -v test - - name: "pypy3" - python: "pypy3.6-7.1.1" - install: - - pip install -e . - script: - - pytest -v test + - name: "Python 2 (fallback)" python: "2.7" - install: - - pip install -U pip - - pip install -U pytest - - pip install . - script: - - pytest -v test + <<: *pure + + - name: "pypy2.7" + python: "pypy2.7-7.1.1" + <<: *pure + + - name: "pypy3" + python: "pypy3.6-7.1.1" + <<: *pure install: - pip install -U pip - - pip install -U pytest - - pip install -r requirements.txt + - pip install -U pytest pytest-cov codecov + - pip install -r requirements.txt # Cython - make cython - pip install -e . script: - python -c 'import sys; print(hex(sys.maxsize))' - python -c 'from msgpack import _cmsgpack' - - pytest -v test - - MSGPACK_PUREPYTHON=x pytest -v test + - pytest --cov=msgpack -v test + - MSGPACK_PUREPYTHON=x pytest --cov=msgpack -v test + +after_success: + - if [ -f .coverage ]; then + codecov; + fi # vim: sw=2 ts=2 From 7a8ce0f9ca910a851b6835d26b1d6970a188fa4e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Dec 2019 20:34:18 +0900 Subject: [PATCH 1017/1172] Remove unused import --- Makefile | 2 +- msgpack/fallback.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a1edc88..f8971cc 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: cython .PHONY: black black: - black . + black msgpack/ test/ .PHONY: cython cython: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index fa2f3a8..388a5ab 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -2,7 +2,6 @@ import sys import struct -import warnings PY2 = sys.version_info[0] == 2 From f6f6f328eb2d7b7f1272fa7addb31d2ac5bef207 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Dec 2019 21:16:27 +0900 Subject: [PATCH 1018/1172] Fix fallback Unpacker.read() (#388) Fixes #352. --- msgpack/fallback.py | 4 +++- test/test_sequnpack.py | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 388a5ab..85a711b 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -357,7 +357,9 @@ class Unpacker(object): return self._buffer[self._buff_i :] def read_bytes(self, n): - return self._read(n) + ret = self._read(n) + self._consume() + return ret def _read(self, n): # (int) -> bytearray diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 9b69479..ad29de8 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -93,6 +93,15 @@ def test_readbytes(): assert unpacker.unpack() == ord(b"a") assert unpacker.unpack() == ord(b"r") + # Issue 352 + u = Unpacker() + u.feed(b"x") + assert bytes(u.read_bytes(1)) == b"x" + with raises(StopIteration): + next(u) + u.feed(b"\1") + assert next(u) == 1 + def test_issue124(): unpacker = Unpacker() From 5ba496c79a45c6938f3e850718349cfa51cc38ae Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Dec 2019 21:23:54 +0900 Subject: [PATCH 1019/1172] Move Black from Travis to Github Actions (#390) --- .github/workflows/black.yaml | 21 +++++++++++++++++++++ .travis.yml | 8 -------- 2 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/black.yaml diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml new file mode 100644 index 0000000..cabd0cc --- /dev/null +++ b/.github/workflows/black.yaml @@ -0,0 +1,21 @@ +name: Black + +on: ["push", "pull_request"] + +jobs: + black: + runs-on: ubuntu-latest + steps: + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: '3.x' + architecture: 'x64' + + - name: Checkout + uses: actions/checkout@v1 + + - name: Black Code Formatter + run: | + pip install black + black --diff --check msgpack/ test/ diff --git a/.travis.yml b/.travis.yml index 378bc80..39db1d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,14 +23,6 @@ _pure: &pure matrix: include: - - name: Black - language: python - python: 3.7 - install: - - pip install black - script: - - black --check --diff . - - name: 32bit build language: python services: From 0fc0eb2f16fcc7d0271792f93a90af389f66dafb Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Dec 2019 21:26:28 +0900 Subject: [PATCH 1020/1172] Update README --- README.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index f9f074f..c13267b 100644 --- a/README.rst +++ b/README.rst @@ -34,19 +34,19 @@ I upload transitional package (msgpack-python 0.5 which depending on msgpack) for smooth transition from msgpack-python to msgpack. Sadly, this doesn't work for upgrade install. After `pip install -U msgpack-python`, -msgpack is removed and `import msgpack` fail. +msgpack is removed, and `import msgpack` fail. -Compatibility with old format -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Compatibility with the old format +^^^^^^^^^^^^^^^^^^^^^^----^^^^^^^ You can use ``use_bin_type=False`` option to pack ``bytes`` -object into raw type in old msgpack spec, instead of bin type in new msgpack spec. +object into raw type in the old msgpack spec, instead of bin type in new msgpack spec. -You can unpack old msgpack formatk using ``raw=True`` option. +You can unpack old msgpack format using ``raw=True`` option. It unpacks str (raw) type in msgpack into Python bytes. -See note in below for detail. +See note below for detail. Install @@ -67,7 +67,7 @@ But msgpack provides a pure Python implementation (``msgpack.fallback``) for PyPy and Python 2. Since the [pip](https://pip.pypa.io/) uses the pure Python implementation, -Python 2 support will not be dropped in foreseeable feature. +Python 2 support will not be dropped in the foreseeable future. Windows From d8e3cf0563989a660398318a7c788645124e1d8b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Dec 2019 22:23:15 +0900 Subject: [PATCH 1021/1172] Make strict_map_key default to True (#392) --- msgpack/_unpacker.pyx | 8 +++----- msgpack/fallback.py | 6 ++---- test/test_case.py | 2 +- test/test_format.py | 2 +- test/test_limits.py | 4 ++-- test/test_obj.py | 9 ++++++--- test/test_pack.py | 6 +++--- test/test_sequnpack.py | 2 +- 8 files changed, 19 insertions(+), 20 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index f10e99d..53ecf86 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -131,7 +131,7 @@ cdef inline int get_data_from_buffer(object obj, def unpackb(object packed, *, object object_hook=None, object list_hook=None, - bint use_list=True, bint raw=False, bint strict_map_key=False, + bint use_list=True, bint raw=False, bint strict_map_key=True, unicode_errors=None, object_pairs_hook=None, ext_hook=ExtType, Py_ssize_t max_str_len=-1, @@ -221,9 +221,7 @@ cdef class Unpacker(object): Otherwise, unpack to Python str by decoding with UTF-8 encoding (default). :param bool strict_map_key: - If true, only str or bytes are accepted for map (dict) keys. - It's False by default for backward-compatibility. - But it will be True from msgpack 1.0. + If true (default), only str or bytes are accepted for map (dict) keys. :param callable object_hook: When specified, it should be callable. @@ -305,7 +303,7 @@ cdef class Unpacker(object): self.buf = NULL def __init__(self, file_like=None, *, Py_ssize_t read_size=0, - bint use_list=True, bint raw=False, bint strict_map_key=False, + bint use_list=True, bint raw=False, bint strict_map_key=True, object object_hook=None, object object_pairs_hook=None, object list_hook=None, unicode_errors=None, Py_ssize_t max_buffer_size=0, object ext_hook=ExtType, diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 85a711b..7df92f5 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -175,9 +175,7 @@ class Unpacker(object): Otherwise, unpack to Python str by decoding with UTF-8 encoding (default). :param bool strict_map_key: - If true, only str or bytes are accepted for map (dict) keys. - It's False by default for backward-compatibility. - But it will be True from msgpack 1.0. + If true (default), only str or bytes are accepted for map (dict) keys. :param callable object_hook: When specified, it should be callable. @@ -249,7 +247,7 @@ class Unpacker(object): read_size=0, use_list=True, raw=False, - strict_map_key=False, + strict_map_key=True, object_hook=None, object_pairs_hook=None, list_hook=None, diff --git a/test/test_case.py b/test/test_case.py index 3e60e59..a0a3c5a 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -92,7 +92,7 @@ def test_array32(): def match(obj, buf): assert packb(obj) == buf - assert unpackb(buf, use_list=0) == obj + assert unpackb(buf, use_list=0, strict_map_key=False) == obj def test_match(): diff --git a/test/test_format.py b/test/test_format.py index 8c2f03f..d455f7c 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -5,7 +5,7 @@ from msgpack import unpackb def check(src, should, use_list=0, raw=True): - assert unpackb(src, use_list=use_list, raw=raw) == should + assert unpackb(src, use_list=use_list, raw=raw, strict_map_key=False) == should def testSimpleValue(): diff --git a/test/test_limits.py b/test/test_limits.py index 6e85030..65e6bcc 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -87,11 +87,11 @@ def test_max_map_len(): d = {1: 2, 3: 4, 5: 6} packed = packb(d) - unpacker = Unpacker(max_map_len=3) + unpacker = Unpacker(max_map_len=3, strict_map_key=False) unpacker.feed(packed) assert unpacker.unpack() == d - unpacker = Unpacker(max_map_len=2) + unpacker = Unpacker(max_map_len=2, strict_map_key=False) with pytest.raises(UnpackValueError): unpacker.feed(packed) unpacker.unpack() diff --git a/test/test_obj.py b/test/test_obj.py index 0b99cea..86c557c 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -33,7 +33,10 @@ def test_decode_pairs_hook(): packed = packb([3, {1: 2, 3: 4}]) prod_sum = 1 * 2 + 3 * 4 unpacked = unpackb( - packed, object_pairs_hook=lambda l: sum(k * v for k, v in l), use_list=1 + packed, + object_pairs_hook=lambda l: sum(k * v for k, v in l), + use_list=1, + strict_map_key=False, ) assert unpacked[1] == prod_sum @@ -70,10 +73,10 @@ def bad_complex_decoder(o): def test_an_exception_in_objecthook1(): with raises(DecodeError): packed = packb({1: {"__complex__": True, "real": 1, "imag": 2}}) - unpackb(packed, object_hook=bad_complex_decoder) + unpackb(packed, object_hook=bad_complex_decoder, strict_map_key=False) def test_an_exception_in_objecthook2(): with raises(DecodeError): packed = packb({1: [{"__complex__": True, "real": 1, "imag": 2}]}) - unpackb(packed, list_hook=bad_complex_decoder, use_list=1) + unpackb(packed, list_hook=bad_complex_decoder, use_list=1, strict_map_key=False) diff --git a/test/test_pack.py b/test/test_pack.py index de212ef..932f760 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -14,7 +14,7 @@ from msgpack import packb, unpackb, Unpacker, Packer, pack def check(data, use_list=False): - re = unpackb(packb(data), use_list=use_list) + re = unpackb(packb(data), use_list=use_list, strict_map_key=False) assert re == data @@ -166,7 +166,7 @@ def testMapSize(sizes=[0, 5, 50, 1000]): bio.write(packer.pack(i * 2)) # value bio.seek(0) - unpacker = Unpacker(bio) + unpacker = Unpacker(bio, strict_map_key=False) for size in sizes: assert unpacker.unpack() == dict((i, i * 2) for i in range(size)) @@ -186,7 +186,7 @@ def test_pairlist(): pairlist = [(b"a", 1), (2, b"b"), (b"foo", b"bar")] packer = Packer() packed = packer.pack_map_pairs(pairlist) - unpacked = unpackb(packed, object_pairs_hook=list) + unpacked = unpackb(packed, object_pairs_hook=list, strict_map_key=False) assert pairlist == unpacked diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index ad29de8..6293a45 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -132,7 +132,7 @@ def test_unpack_tell(): pack(m, stream) offsets.append(stream.tell()) stream.seek(0) - unpacker = Unpacker(stream) + unpacker = Unpacker(stream, strict_map_key=False) for m, o in zip(messages, offsets): m2 = next(unpacker) assert m == m2 From 5399f8180d23c147b1243d7c39aa19f9a8ba840a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 9 Dec 2019 17:02:35 +0900 Subject: [PATCH 1022/1172] Update README (#393) --- README.rst | 55 +++++++++++++++++++++++++++++++++++---------- msgpack/fallback.py | 4 ++-- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index c13267b..d01e963 100644 --- a/README.rst +++ b/README.rst @@ -38,7 +38,7 @@ msgpack is removed, and `import msgpack` fail. Compatibility with the old format -^^^^^^^^^^^^^^^^^^^^^^----^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can use ``use_bin_type=False`` option to pack ``bytes`` object into raw type in the old msgpack spec, instead of bin type in new msgpack spec. @@ -49,6 +49,32 @@ It unpacks str (raw) type in msgpack into Python bytes. See note below for detail. +Major breaking changes in msgpack 1.0 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* Python 2 + + * The extension module does not support Python 2 anymore. + The pure Python implementation (``msgpack.fallback``) is used for Python 2. + +* Packer + + * ``use_bin_type=True`` by default. bytes are encoded in bin type in msgpack. + **If you are still sing Python 2, you must use unicode for all string types.** + You can use ``use_bin_type=False`` to encode into old msgpack format. + * ``encoding`` option is removed. UTF-8 is used always. + +* Unpacker + + * ``raw=False`` by default. It assumes str types are valid UTF-8 string + and decode them to Python str (unicode) object. + * ``encdoding`` option is rmeoved. You can use ``raw=True`` to support old format. + * Default value of ``max_buffer_size`` is changed from 0 to 100 MiB. + * Default value of ``strict_map_key`` is changed to True to avoid hashdos. + You need to pass ``strict_map_key=False`` if you have data which contain map keys + which type is not bytes or str. + + Install ------- @@ -270,27 +296,32 @@ To use the **ext** type, pass ``msgpack.ExtType`` object to packer. You can use it with ``default`` and ``ext_hook``. See below. -Note about performance ----------------------- +Security +^^^^^^^^ -GC -^^ +To unpacking data received from unreliable source, msgpack provides +two security options. + +``max_buffer_size`` (default: 100*1024*1024) limits the internal buffer size. +It is used to limit the preallocated list size too. + +``strict_map_key`` (default: ``True``) limits the type of map keys to bytes and str. +While msgpack spec doesn't limit the types of the map keys, +there is a risk of the hashdos. +If you need to support other types for map keys, use ``strict_map_key=False``. + + +Performance tips +^^^^^^^^^^^^^^^^ CPython's GC starts when growing allocated object. This means unpacking may cause useless GC. You can use ``gc.disable()`` when unpacking large message. -use_list option -^^^^^^^^^^^^^^^ - List is the default sequence type of Python. But tuple is lighter than list. You can use ``use_list=False`` while unpacking when performance is important. -Python's dict can't use list as key and MessagePack allows array for key of mapping. -``use_list=False`` allows unpacking such message. -Another way to unpacking such object is using ``object_pairs_hook``. - Development ----------- diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 7df92f5..3704f9d 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -747,7 +747,7 @@ class Packer(object): :param bool strict_types: If set to true, types will be checked to be exact. Derived classes - from serializeable types will not be serialized and will be + from serializable types will not be serialized and will be treated as unsupported type and forwarded to default. Additionally tuples will not be serialized as lists. This is useful when trying to implement accurate serialization @@ -1014,7 +1014,7 @@ class Packer(object): def reset(self): """Reset internal buffer. - This method is usaful only when autoreset=False. + This method is useful only when autoreset=False. """ self._buffer = StringIO() From c356035a576c38db5ca232ede07b291087f1b8b2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 9 Dec 2019 17:03:12 +0900 Subject: [PATCH 1023/1172] Unpacker: Change max_buffer_size to 100MiB (#391) --- msgpack/_unpacker.pyx | 42 ++++++++++++++++++++++-------------------- msgpack/fallback.py | 33 ++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 35 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 53ecf86..0ff633b 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -234,27 +234,28 @@ cdef class Unpacker(object): (See also simplejson) :param int max_buffer_size: - Limits size of data waiting unpacked. 0 means system's INT_MAX (default). + Limits size of data waiting unpacked. 0 means system's INT_MAX. + The default value is 100*1024*1024 (100MiB). Raises `BufferFull` exception when it is insufficient. You should set this parameter when unpacking data from untrusted source. :param int max_str_len: Deprecated, use *max_buffer_size* instead. - Limits max length of str. (default: max_buffer_size or 1024*1024) + Limits max length of str. (default: max_buffer_size) :param int max_bin_len: Deprecated, use *max_buffer_size* instead. - Limits max length of bin. (default: max_buffer_size or 1024*1024) + Limits max length of bin. (default: max_buffer_size) :param int max_array_len: - Limits max length of array. (default: max_buffer_size or 128*1024) + Limits max length of array. (default: max_buffer_size) :param int max_map_len: - Limits max length of map. (default: max_buffer_size//2 or 32*1024) + Limits max length of map. (default: max_buffer_size//2) :param int max_ext_len: Deprecated, use *max_buffer_size* instead. - Limits max size of ext type. (default: max_buffer_size or 1024*1024) + Limits max size of ext type. (default: max_buffer_size) :param str unicode_errors: Error handler used for decoding str type. (default: `'strict'`) @@ -262,13 +263,13 @@ cdef class Unpacker(object): Example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024) + unpacker = Unpacker(file_like) for o in unpacker: process(o) Example of streaming deserialize from socket:: - unpacker = Unpacker(max_buffer_size=10*1024*1024) + unpacker = Unpacker() while True: buf = sock.recv(1024**2) if not buf: @@ -305,7 +306,7 @@ cdef class Unpacker(object): def __init__(self, file_like=None, *, Py_ssize_t read_size=0, bint use_list=True, bint raw=False, bint strict_map_key=True, object object_hook=None, object object_pairs_hook=None, object list_hook=None, - unicode_errors=None, Py_ssize_t max_buffer_size=0, + unicode_errors=None, Py_ssize_t max_buffer_size=100*1024*1024, object ext_hook=ExtType, Py_ssize_t max_str_len=-1, Py_ssize_t max_bin_len=-1, @@ -325,23 +326,24 @@ cdef class Unpacker(object): if not PyCallable_Check(self.file_like_read): raise TypeError("`file_like.read` must be a callable.") - if max_str_len == -1: - max_str_len = max_buffer_size or 1024*1024 - if max_bin_len == -1: - max_bin_len = max_buffer_size or 1024*1024 - if max_array_len == -1: - max_array_len = max_buffer_size or 128*1024 - if max_map_len == -1: - max_map_len = max_buffer_size//2 or 32*1024 - if max_ext_len == -1: - max_ext_len = max_buffer_size or 1024*1024 - if not max_buffer_size: max_buffer_size = INT_MAX + if max_str_len == -1: + max_str_len = max_buffer_size + if max_bin_len == -1: + max_bin_len = max_buffer_size + if max_array_len == -1: + max_array_len = max_buffer_size + if max_map_len == -1: + max_map_len = max_buffer_size//2 + if max_ext_len == -1: + max_ext_len = max_buffer_size + if read_size > max_buffer_size: raise ValueError("read_size should be less or equal to max_buffer_size") if not read_size: read_size = min(max_buffer_size, 1024**2) + self.max_buffer_size = max_buffer_size self.read_size = read_size self.buf = PyMem_Malloc(read_size) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 3704f9d..f6ba424 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -193,39 +193,40 @@ class Unpacker(object): contains invalid UTF-8 string. :param int max_buffer_size: - Limits size of data waiting unpacked. 0 means system's INT_MAX (default). + Limits size of data waiting unpacked. 0 means 2**32-1. + The default value is 100*1024*1024 (100MiB). Raises `BufferFull` exception when it is insufficient. You should set this parameter when unpacking data from untrusted source. :param int max_str_len: Deprecated, use *max_buffer_size* instead. - Limits max length of str. (default: max_buffer_size or 1024*1024) + Limits max length of str. (default: max_buffer_size) :param int max_bin_len: Deprecated, use *max_buffer_size* instead. - Limits max length of bin. (default: max_buffer_size or 1024*1024) + Limits max length of bin. (default: max_buffer_size) :param int max_array_len: Limits max length of array. - (default: max_buffer_size or 128*1024) + (default: max_buffer_size) :param int max_map_len: Limits max length of map. - (default: max_buffer_size//2 or 32*1024) + (default: max_buffer_size//2) :param int max_ext_len: Deprecated, use *max_buffer_size* instead. - Limits max size of ext type. (default: max_buffer_size or 1024*1024) + Limits max size of ext type. (default: max_buffer_size) Example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024) + unpacker = Unpacker(file_like) for o in unpacker: process(o) Example of streaming deserialize from socket:: - unpacker = Unpacker(max_buffer_size=10*1024*1024) + unpacker = Unpacker(max_buffer_size) while True: buf = sock.recv(1024**2) if not buf: @@ -252,7 +253,7 @@ class Unpacker(object): object_pairs_hook=None, list_hook=None, unicode_errors=None, - max_buffer_size=0, + max_buffer_size=100 * 1024 * 1024, ext_hook=ExtType, max_str_len=-1, max_bin_len=-1, @@ -285,18 +286,20 @@ class Unpacker(object): # state, which _buf_checkpoint records. self._buf_checkpoint = 0 + if not max_buffer_size: + max_buffer_size = 2 ** 31 - 1 if max_str_len == -1: - max_str_len = max_buffer_size or 1024 * 1024 + max_str_len = max_buffer_size if max_bin_len == -1: - max_bin_len = max_buffer_size or 1024 * 1024 + max_bin_len = max_buffer_size if max_array_len == -1: - max_array_len = max_buffer_size or 128 * 1024 + max_array_len = max_buffer_size if max_map_len == -1: - max_map_len = max_buffer_size // 2 or 32 * 1024 + max_map_len = max_buffer_size // 2 if max_ext_len == -1: - max_ext_len = max_buffer_size or 1024 * 1024 + max_ext_len = max_buffer_size - self._max_buffer_size = max_buffer_size or 2 ** 31 - 1 + self._max_buffer_size = max_buffer_size if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") self._read_size = read_size or min(self._max_buffer_size, 16 * 1024) From d10f12db8f328130a13df759bc9cb3fa064cc8b8 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 9 Dec 2019 18:12:51 +0900 Subject: [PATCH 1024/1172] typo --- msgpack/_packer.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 8cf3c05..2a768b0 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -341,7 +341,7 @@ cdef class Packer(object): def reset(self): """Reset internal buffer. - This method is usaful only when autoreset=False. + This method is useful only when autoreset=False. """ self.pk.length = 0 From 5fd611909319d03200774ea3c7a6ae16dbd26c12 Mon Sep 17 00:00:00 2001 From: Marty B Date: Mon, 9 Dec 2019 11:29:47 +0100 Subject: [PATCH 1025/1172] Simplify check for bool type (#362) --- msgpack/_packer.pyx | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 2a768b0..1426439 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -153,11 +153,10 @@ cdef class Packer(object): while True: if o is None: ret = msgpack_pack_nil(&self.pk) - elif PyBool_Check(o) if strict_types else isinstance(o, bool): - if o: - ret = msgpack_pack_true(&self.pk) - else: - ret = msgpack_pack_false(&self.pk) + elif o is True: + ret = msgpack_pack_true(&self.pk) + elif o is False: + ret = msgpack_pack_false(&self.pk) elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o): # PyInt_Check(long) is True for Python 3. # So we should test long before int. From 2186455d1579affc33253484d9445f7bdf3f7c29 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 11 Dec 2019 23:48:16 +0900 Subject: [PATCH 1026/1172] Support datetime. (#394) --- Makefile | 2 +- msgpack/_cmsgpack.pyx | 7 ++++ msgpack/_packer.pyx | 19 +++++++++-- msgpack/_unpacker.pyx | 23 ++++++++++---- msgpack/ext.py | 27 +++++++++++++++- msgpack/fallback.py | 48 +++++++++++++++++++++++++--- msgpack/unpack.h | 72 ++++++++++++++++++++++++++++++++++++++---- test/test_timestamp.py | 48 ++++++++++++++++++++++++++-- 8 files changed, 222 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index f8971cc..e2f25cf 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: cython .PHONY: black black: - black msgpack/ test/ + black msgpack/ test/ setup.py .PHONY: cython cython: diff --git a/msgpack/_cmsgpack.pyx b/msgpack/_cmsgpack.pyx index 8ebdbf5..1faaac3 100644 --- a/msgpack/_cmsgpack.pyx +++ b/msgpack/_cmsgpack.pyx @@ -1,4 +1,11 @@ # coding: utf-8 #cython: embedsignature=True, c_string_encoding=ascii, language_level=3 +from cpython.datetime cimport import_datetime, datetime_new +import_datetime() + +import datetime +cdef object utc = datetime.timezone.utc +cdef object epoch = datetime_new(1970, 1, 1, 0, 0, 0, 0, tz=utc) + include "_packer.pyx" include "_unpacker.pyx" diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 1426439..b470646 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -2,6 +2,10 @@ from cpython cimport * from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact +from cpython.datetime cimport ( + PyDateTime_CheckExact, PyDelta_CheckExact, + datetime_tzinfo, timedelta_days, timedelta_seconds, timedelta_microseconds, +) cdef ExtType cdef Timestamp @@ -99,8 +103,9 @@ cdef class Packer(object): cdef object _berrors cdef const char *unicode_errors cdef bint strict_types - cdef bool use_float + cdef bint use_float cdef bint autoreset + cdef bint datetime def __cinit__(self): cdef int buf_size = 1024*1024 @@ -110,12 +115,13 @@ cdef class Packer(object): self.pk.buf_size = buf_size self.pk.length = 0 - def __init__(self, *, default=None, unicode_errors=None, + def __init__(self, *, default=None, bint use_single_float=False, bint autoreset=True, bint use_bin_type=True, - bint strict_types=False): + bint strict_types=False, bint datetime=False, unicode_errors=None): self.use_float = use_single_float self.strict_types = strict_types self.autoreset = autoreset + self.datetime = datetime self.pk.use_bin_type = use_bin_type if default is not None: if not PyCallable_Check(default): @@ -262,6 +268,13 @@ cdef class Packer(object): if ret == 0: ret = msgpack_pack_raw_body(&self.pk, view.buf, L) PyBuffer_Release(&view); + elif self.datetime and PyDateTime_CheckExact(o) and datetime_tzinfo(o) is not None: + delta = o - epoch + if not PyDelta_CheckExact(delta): + raise ValueError("failed to calculate delta") + llval = timedelta_days(delta) * (24*60*60) + timedelta_seconds(delta) + ulval = timedelta_microseconds(delta) * 1000 + ret = msgpack_pack_timestamp(&self.pk, llval, ulval) elif not default_used and self._default: o = self._default(o) default_used = 1 diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 0ff633b..43c93a2 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -1,7 +1,6 @@ # coding: utf-8 from cpython cimport * - cdef extern from "Python.h": ctypedef struct PyObject cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1 @@ -21,6 +20,8 @@ from .exceptions import ( ) from .ext import ExtType, Timestamp +cdef object giga = 1_000_000_000 + cdef extern from "unpack.h": ctypedef struct msgpack_user: @@ -28,10 +29,13 @@ cdef extern from "unpack.h": bint raw bint has_pairs_hook # call object_hook with k-v pairs bint strict_map_key + int timestamp PyObject* object_hook PyObject* list_hook PyObject* ext_hook PyObject* timestamp_t + PyObject *giga; + PyObject *utc; char *unicode_errors Py_ssize_t max_str_len Py_ssize_t max_bin_len @@ -57,7 +61,8 @@ cdef extern from "unpack.h": cdef inline init_ctx(unpack_context *ctx, object object_hook, object object_pairs_hook, object list_hook, object ext_hook, - bint use_list, bint raw, bint strict_map_key, + bint use_list, bint raw, int timestamp, + bint strict_map_key, const char* unicode_errors, Py_ssize_t max_str_len, Py_ssize_t max_bin_len, Py_ssize_t max_array_len, Py_ssize_t max_map_len, @@ -99,8 +104,14 @@ cdef inline init_ctx(unpack_context *ctx, raise TypeError("ext_hook must be a callable.") ctx.user.ext_hook = ext_hook + if timestamp < 0 or 3 < timestamp: + raise ValueError("timestamp must be 0..3") + # Add Timestamp type to the user object so it may be used in unpack.h + ctx.user.timestamp = timestamp ctx.user.timestamp_t = Timestamp + ctx.user.giga = giga + ctx.user.utc = utc ctx.user.unicode_errors = unicode_errors def default_read_extended_type(typecode, data): @@ -131,7 +142,7 @@ cdef inline int get_data_from_buffer(object obj, def unpackb(object packed, *, object object_hook=None, object list_hook=None, - bint use_list=True, bint raw=False, bint strict_map_key=True, + bint use_list=True, bint raw=False, int timestamp=0, bint strict_map_key=True, unicode_errors=None, object_pairs_hook=None, ext_hook=ExtType, Py_ssize_t max_str_len=-1, @@ -179,7 +190,7 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, try: init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook, - use_list, raw, strict_map_key, cerr, + use_list, raw, timestamp, strict_map_key, cerr, max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) ret = unpack_construct(&ctx, buf, buf_len, &off) finally: @@ -304,7 +315,7 @@ cdef class Unpacker(object): self.buf = NULL def __init__(self, file_like=None, *, Py_ssize_t read_size=0, - bint use_list=True, bint raw=False, bint strict_map_key=True, + bint use_list=True, bint raw=False, int timestamp=0, bint strict_map_key=True, object object_hook=None, object object_pairs_hook=None, object list_hook=None, unicode_errors=None, Py_ssize_t max_buffer_size=100*1024*1024, object ext_hook=ExtType, @@ -359,7 +370,7 @@ cdef class Unpacker(object): cerr = unicode_errors init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, - ext_hook, use_list, raw, strict_map_key, cerr, + ext_hook, use_list, raw, timestamp, strict_map_key, cerr, max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) diff --git a/msgpack/ext.py b/msgpack/ext.py index c7efff6..09adb34 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -1,12 +1,18 @@ # coding: utf-8 from collections import namedtuple +import datetime import sys import struct PY2 = sys.version_info[0] == 2 + if not PY2: long = int + try: + _utc = datetime.timezone.utc + except AttributeError: + _utc = datetime.timezone(datetime.timedelta(0)) class ExtType(namedtuple("ExtType", "code data")): @@ -131,7 +137,7 @@ class Timestamp(object): data = struct.pack("!Iq", self.nanoseconds, self.seconds) return data - def to_float_s(self): + def to_float(self): """Get the timestamp as a floating-point value. :returns: posix timestamp @@ -139,6 +145,12 @@ class Timestamp(object): """ return self.seconds + self.nanoseconds / 1e9 + @staticmethod + def from_float(unix_float): + seconds = int(unix_float) + nanoseconds = int((unix_float % 1) * 1000000000) + return Timestamp(seconds, nanoseconds) + def to_unix_ns(self): """Get the timestamp as a unixtime in nanoseconds. @@ -146,3 +158,16 @@ class Timestamp(object): :rtype: int """ return int(self.seconds * 1e9 + self.nanoseconds) + + if not PY2: + + def to_datetime(self): + """Get the timestamp as a UTC datetime. + + :rtype: datetime. + """ + return datetime.datetime.fromtimestamp(self.to_float(), _utc) + + @staticmethod + def from_datetime(dt): + return Timestamp.from_float(dt.timestamp()) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index f6ba424..9ba98bf 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -1,5 +1,6 @@ """Fallback pure Python implementation of msgpack""" +from datetime import datetime as _DateTime import sys import struct @@ -174,6 +175,14 @@ class Unpacker(object): If true, unpack msgpack raw to Python bytes. Otherwise, unpack to Python str by decoding with UTF-8 encoding (default). + :param int timestamp: + Control how timestamp type is unpacked: + + 0 - Tiemstamp + 1 - float (Seconds from the EPOCH) + 2 - int (Nanoseconds from the EPOCH) + 3 - datetime.datetime (UTC). Python 2 is not supported. + :param bool strict_map_key: If true (default), only str or bytes are accepted for map (dict) keys. @@ -248,6 +257,7 @@ class Unpacker(object): read_size=0, use_list=True, raw=False, + timestamp=0, strict_map_key=True, object_hook=None, object_pairs_hook=None, @@ -307,6 +317,9 @@ class Unpacker(object): self._strict_map_key = bool(strict_map_key) self._unicode_errors = unicode_errors self._use_list = use_list + if not (0 <= timestamp <= 3): + raise ValueError("timestamp must be 0..3") + self._timestamp = timestamp self._list_hook = list_hook self._object_hook = object_hook self._object_pairs_hook = object_pairs_hook @@ -672,10 +685,21 @@ class Unpacker(object): else: obj = obj.decode("utf_8", self._unicode_errors) return obj - if typ == TYPE_EXT: - return self._ext_hook(n, bytes(obj)) if typ == TYPE_BIN: return bytes(obj) + if typ == TYPE_EXT: + if n == -1: # timestamp + ts = Timestamp.from_bytes(bytes(obj)) + if self._timestamp == 1: + return ts.to_float() + elif self._timestamp == 2: + return ts.to_unix_ns() + elif self._timestamp == 3: + return ts.to_datetime() + else: + return ts + else: + return self._ext_hook(n, bytes(obj)) assert typ == TYPE_IMMEDIATE return obj @@ -756,6 +780,12 @@ class Packer(object): This is useful when trying to implement accurate serialization for python types. + :param bool datetime: + If set to true, datetime with tzinfo is packed into Timestamp type. + Note that the tzinfo is stripped in the timestamp. + You can get UTC datetime with `timestamp=3` option of the Unapcker. + (Python 2 is not supported). + :param str unicode_errors: The error handler for encoding unicode. (default: 'strict') DO NOT USE THIS!! This option is kept for very specific usage. @@ -764,18 +794,22 @@ class Packer(object): def __init__( self, default=None, - unicode_errors=None, use_single_float=False, autoreset=True, use_bin_type=True, strict_types=False, + datetime=False, + unicode_errors=None, ): self._strict_types = strict_types self._use_float = use_single_float self._autoreset = autoreset self._use_bin_type = use_bin_type - self._unicode_errors = unicode_errors or "strict" self._buffer = StringIO() + if PY2 and datetime: + raise ValueError("datetime is not supported in Python 2") + self._datetime = bool(datetime) + self._unicode_errors = unicode_errors or "strict" if default is not None: if not callable(default): raise TypeError("default must be callable") @@ -891,6 +925,12 @@ class Packer(object): return self._pack_map_pairs( len(obj), dict_iteritems(obj), nest_limit - 1 ) + + if self._datetime and check(obj, _DateTime): + obj = Timestamp.from_datetime(obj) + default_used = 1 + continue + if not default_used and self._default is not None: obj = self._default(obj) default_used = 1 diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 4380ec5..debdf71 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -24,10 +24,13 @@ typedef struct unpack_user { bool raw; bool has_pairs_hook; bool strict_map_key; + int timestamp; PyObject *object_hook; PyObject *list_hook; PyObject *ext_hook; PyObject *timestamp_t; + PyObject *giga; + PyObject *utc; const char *unicode_errors; Py_ssize_t max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len; } unpack_user; @@ -268,7 +271,7 @@ typedef struct msgpack_timestamp { /* * Unpack ext buffer to a timestamp. Pulled from msgpack-c timestamp.h. */ -static inline int unpack_timestamp(const char* buf, unsigned int buflen, msgpack_timestamp* ts) { +static int unpack_timestamp(const char* buf, unsigned int buflen, msgpack_timestamp* ts) { switch (buflen) { case 4: ts->tv_nsec = 0; @@ -292,10 +295,11 @@ static inline int unpack_timestamp(const char* buf, unsigned int buflen, msgpack } } -static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos, - unsigned int length, msgpack_unpack_object* o) +#include "datetime.h" + +static int unpack_callback_ext(unpack_user* u, const char* base, const char* pos, + unsigned int length, msgpack_unpack_object* o) { - PyObject *py; int8_t typecode = (int8_t)*pos++; if (!u->ext_hook) { PyErr_SetString(PyExc_AssertionError, "u->ext_hook cannot be NULL"); @@ -305,13 +309,67 @@ static inline int unpack_callback_ext(unpack_user* u, const char* base, const ch PyErr_Format(PyExc_ValueError, "%u exceeds max_ext_len(%zd)", length, u->max_ext_len); return -1; } + + PyObject *py = NULL; // length also includes the typecode, so the actual data is length-1 if (typecode == -1) { msgpack_timestamp ts; - if (unpack_timestamp(pos, length-1, &ts) == 0) { + if (unpack_timestamp(pos, length-1, &ts) < 0) { + return -1; + } + + if (u->timestamp == 2) { // int + PyObject *a = PyLong_FromLongLong(ts.tv_sec); + if (a == NULL) return -1; + + PyObject *c = PyNumber_Multiply(a, u->giga); + Py_DECREF(a); + if (c == NULL) { + return -1; + } + + PyObject *b = PyLong_FromUnsignedLong(ts.tv_nsec); + if (b == NULL) { + Py_DECREF(c); + return -1; + } + + py = PyNumber_Add(c, b); + Py_DECREF(c); + Py_DECREF(b); + } + else if (u->timestamp == 0) { // Timestamp py = PyObject_CallFunction(u->timestamp_t, "(Lk)", ts.tv_sec, ts.tv_nsec); - } else { - py = NULL; + } + else { // float or datetime + PyObject *a = PyFloat_FromDouble((double)ts.tv_nsec); + if (a == NULL) return -1; + + PyObject *b = PyNumber_TrueDivide(a, u->giga); + Py_DECREF(a); + if (b == NULL) return -1; + + PyObject *c = PyLong_FromLongLong(ts.tv_sec); + if (c == NULL) { + Py_DECREF(b); + return -1; + } + + a = PyNumber_Add(b, c); + Py_DECREF(b); + Py_DECREF(c); + + if (u->timestamp == 3) { // datetime + PyObject *t = PyTuple_Pack(2, a, u->utc); + Py_DECREF(a); + if (t == NULL) { + return -1; + } + py = PyDateTime_FromTimestamp(t); + Py_DECREF(t); + } else { // float + py = a; + } } } else { py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1); diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 1348e69..822994c 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -1,5 +1,11 @@ +import pytest +import sys +import datetime import msgpack -from msgpack import Timestamp +from msgpack.ext import Timestamp + +if sys.version_info[0] > 2: + from msgpack.ext import _utc def test_timestamp(): @@ -42,5 +48,43 @@ def test_timestamp(): def test_timestamp_to(): t = Timestamp(42, 14) - assert t.to_float_s() == 42.000000014 + assert t.to_float() == 42.000000014 assert t.to_unix_ns() == 42000000014 + + +@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") +def test_timestamp_datetime(): + t = Timestamp(42, 14) + assert t.to_datetime() == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc) + + +@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") +def test_unpack_datetime(): + t = Timestamp(42, 14) + packed = msgpack.packb(t) + unpacked = msgpack.unpackb(packed, timestamp=3) + assert unpacked == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc) + + +@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") +def test_pack_datetime(): + t = Timestamp(42, 14000) + dt = t.to_datetime() + assert dt == datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=_utc) + + packed = msgpack.packb(dt, datetime=True) + packed2 = msgpack.packb(t) + assert packed == packed2 + + unpacked = msgpack.unpackb(packed) + print(packed, unpacked) + assert unpacked == t + + unpacked = msgpack.unpackb(packed, timestamp=3) + assert unpacked == dt + + x = [] + packed = msgpack.packb(dt, datetime=False, default=x.append) + assert x + assert x[0] == dt + assert msgpack.unpackb(packed) is None From c60e6c7a6ff1815083bf6803ec70f3ac34aaf3bb Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 12 Dec 2019 18:09:07 +0900 Subject: [PATCH 1027/1172] Update README --- ChangeLog.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ChangeLog.rst b/ChangeLog.rst index d44b36a..c70b966 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -6,6 +6,14 @@ Release Date: TBD * Remove Python 2 support from the ``msgpack/_cmsgpack``. ``msgpack/fallback`` still supports Python 2. * Remove ``encoding`` option from the Packer and Unpacker. +* Unpacker: The default value of ``max_buffer_type`` is changed to 100MiB. +* Unpacker: ``strict_map_key`` is True by default now. +* Unpacker: String map keys are interned. +* Drop old buffer protocol support. +* Support Timestamp type. +* Support serializing and decerializing ``datetime`` object + with tzinfo. +* Unpacker: ``Fix Unpacker.read_bytes()`` in fallback implementation. (#352) 0.6.2 From 3df431cafd82354e61b39afd6094003e9c313c43 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 12 Dec 2019 18:25:38 +0900 Subject: [PATCH 1028/1172] Prepare 1.0rc1 --- MANIFEST.in | 2 +- README.rst => README.md | 125 ++++++++++++++-------------------------- msgpack/_version.py | 2 +- setup.py | 6 +- 4 files changed, 49 insertions(+), 86 deletions(-) rename README.rst => README.md (83%) diff --git a/MANIFEST.in b/MANIFEST.in index e1912ca..57d84a4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include setup.py include COPYING -include README.rst +include README.md recursive-include msgpack *.h *.c *.pyx *.cpp recursive-include test *.py diff --git a/README.rst b/README.md similarity index 83% rename from README.rst rename to README.md index d01e963..897a932 100644 --- a/README.rst +++ b/README.md @@ -1,18 +1,9 @@ -====================== -MessagePack for Python -====================== +# MessagePack for Python -.. image:: https://travis-ci.org/msgpack/msgpack-python.svg?branch=master - :target: https://travis-ci.org/msgpack/msgpack-python - :alt: Build Status +[![Build Status](https://travis-ci.org/msgpack/msgpack-python.svg?branch=master)](https://travis-ci.org/msgpack/msgpack-python) +[![Documentation Status](https://readthedocs.org/projects/msgpack-python/badge/?version=latest)](https://msgpack-python.readthedocs.io/en/latest/?badge=latest) -.. image:: https://readthedocs.org/projects/msgpack-python/badge/?version=latest - :target: https://msgpack-python.readthedocs.io/en/latest/?badge=latest - :alt: Documentation Status - - -What's this ------------ +## What's this `MessagePack `_ is an efficient binary serialization format. It lets you exchange data among multiple languages like JSON. @@ -20,11 +11,9 @@ But it's faster and smaller. This package provides CPython bindings for reading and writing MessagePack data. -Very important notes for existing users ---------------------------------------- +## Very important notes for existing users -PyPI package name -^^^^^^^^^^^^^^^^^ +### PyPI package name TL;DR: When upgrading from msgpack-0.4 or earlier, don't do `pip install -U msgpack-python`. Do `pip uninstall msgpack-python; pip install msgpack` instead. @@ -37,8 +26,7 @@ Sadly, this doesn't work for upgrade install. After `pip install -U msgpack-pyt msgpack is removed, and `import msgpack` fail. -Compatibility with the old format -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +### Compatibility with the old format You can use ``use_bin_type=False`` option to pack ``bytes`` object into raw type in the old msgpack spec, instead of bin type in new msgpack spec. @@ -49,8 +37,7 @@ It unpacks str (raw) type in msgpack into Python bytes. See note below for detail. -Major breaking changes in msgpack 1.0 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +### Major breaking changes in msgpack 1.0 * Python 2 @@ -75,16 +62,13 @@ Major breaking changes in msgpack 1.0 which type is not bytes or str. -Install -------- +## Install -:: $ pip install msgpack -Pure Python implementation -^^^^^^^^^^^^^^^^^^^^^^^^^^ +### Pure Python implementation The extension module in msgpack (``msgpack._cmsgpack``) does not support Python 2 and PyPy. @@ -96,26 +80,20 @@ Since the [pip](https://pip.pypa.io/) uses the pure Python implementation, Python 2 support will not be dropped in the foreseeable future. -Windows -^^^^^^^ +### Windows When you can't use a binary distribution, you need to install Visual Studio or Windows SDK on Windows. Without extension, using pure Python implementation on CPython runs slowly. -How to use ----------- +## How to use -.. note:: - - In examples below, I use ``raw=False`` and ``use_bin_type=True`` for users - using msgpack < 1.0. - These options are default from msgpack 1.0 so you can omit them. +NOTE: In examples below, I use ``raw=False`` and ``use_bin_type=True`` for users +using msgpack < 1.0. These options are default from msgpack 1.0 so you can omit them. -One-shot pack & unpack -^^^^^^^^^^^^^^^^^^^^^^ +### One-shot pack & unpack Use ``packb`` for packing and ``unpackb`` for unpacking. msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with @@ -124,20 +102,20 @@ msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with ``pack`` and ``dump`` packs to a file-like object. ``unpack`` and ``load`` unpacks from a file-like object. -.. code-block:: pycon - +```pycon >>> import msgpack >>> msgpack.packb([1, 2, 3], use_bin_type=True) '\x93\x01\x02\x03' >>> msgpack.unpackb(_, raw=False) [1, 2, 3] +``` ``unpack`` unpacks msgpack's array to Python's list, but can also unpack to tuple: -.. code-block:: pycon - +```pycon >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False) (1, 2, 3) +``` You should always specify the ``use_list`` keyword argument for backward compatibility. See performance issues relating to `use_list option`_ below. @@ -145,14 +123,12 @@ See performance issues relating to `use_list option`_ below. Read the docstring for other options. -Streaming unpacking -^^^^^^^^^^^^^^^^^^^ +### Streaming unpacking ``Unpacker`` is a "streaming unpacker". It unpacks multiple objects from one stream (or from bytes provided through its ``feed`` method). -.. code-block:: python - +```py import msgpack from io import BytesIO @@ -165,16 +141,15 @@ stream (or from bytes provided through its ``feed`` method). unpacker = msgpack.Unpacker(buf, raw=False) for unpacked in unpacker: print(unpacked) +``` -Packing/unpacking of custom data type -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +### Packing/unpacking of custom data type It is also possible to pack/unpack custom data types. Here is an example for ``datetime.datetime``. -.. code-block:: python - +```py import datetime import msgpack @@ -196,19 +171,18 @@ It is also possible to pack/unpack custom data types. Here is an example for packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True) this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False) +``` ``Unpacker``'s ``object_hook`` callback receives a dict; the ``object_pairs_hook`` callback may instead be used to receive a list of key-value pairs. -Extended types -^^^^^^^^^^^^^^ +### Extended types It is also possible to pack/unpack custom data types using the **ext** type. -.. code-block:: pycon - +```pycon >>> import msgpack >>> import array >>> def default(obj): @@ -228,10 +202,10 @@ It is also possible to pack/unpack custom data types using the **ext** type. >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False) >>> data == unpacked True +``` -Advanced unpacking control -^^^^^^^^^^^^^^^^^^^^^^^^^^ +### Advanced unpacking control As an alternative to iteration, ``Unpacker`` objects provide ``unpack``, ``skip``, ``read_array_header`` and ``read_map_header`` methods. The former two @@ -243,8 +217,7 @@ in a map, can be unpacked or skipped individually. Each of these methods may optionally write the packed data it reads to a callback function: -.. code-block:: python - +```py from io import BytesIO def distribute(unpacker, get_worker): @@ -258,13 +231,11 @@ callback function: bytestream = BytesIO() unpacker.skip(bytestream.write) worker.send(bytestream.getvalue()) +``` +## Notes -Notes ------ - -string and binary type -^^^^^^^^^^^^^^^^^^^^^^ +### string and binary type Early versions of msgpack didn't distinguish string and binary types. The type for representing both string and binary types was named **raw**. @@ -272,32 +243,29 @@ The type for representing both string and binary types was named **raw**. You can pack into and unpack from this old spec using ``use_bin_type=False`` and ``raw=True`` options. -.. code-block:: pycon - +```pycon >>> import msgpack >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=False), raw=True) [b'spam', b'eggs'] >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), raw=False) [b'spam', 'eggs'] +``` - -ext type -^^^^^^^^ +### ext type To use the **ext** type, pass ``msgpack.ExtType`` object to packer. -.. code-block:: pycon - +```pycon >>> import msgpack >>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy')) >>> msgpack.unpackb(packed) ExtType(code=42, data='xyzzy') +``` You can use it with ``default`` and ``ext_hook``. See below. -Security -^^^^^^^^ +### Security To unpacking data received from unreliable source, msgpack provides two security options. @@ -311,8 +279,7 @@ there is a risk of the hashdos. If you need to support other types for map keys, use ``strict_map_key=False``. -Performance tips -^^^^^^^^^^^^^^^^ +### Performance tips CPython's GC starts when growing allocated object. This means unpacking may cause useless GC. @@ -323,17 +290,13 @@ But tuple is lighter than list. You can use ``use_list=False`` while unpacking when performance is important. -Development ------------ +## Development -Test -^^^^ +### Test MessagePack uses `pytest` for testing. Run test with following command: +``` $ make test - - -.. - vim: filetype=rst +``` diff --git a/msgpack/_version.py b/msgpack/_version.py index 1e73a00..5762e8c 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (0, 6, 2) +version = (1, 0, 0, 'rc1') diff --git a/setup.py b/setup.py index a8c2306..ac0dc30 100755 --- a/setup.py +++ b/setup.py @@ -106,7 +106,7 @@ del libraries, macros desc = "MessagePack (de)serializer." -with io.open("README.rst", encoding="utf-8") as f: +with io.open("README.md", encoding="utf-8") as f: long_desc = f.read() del f @@ -118,7 +118,7 @@ if TRANSITIONAL: setup( name=name, - author="INADA Naoki", + author="Inada Naoki", author_email="songofacandy@gmail.com", version=version_str, cmdclass={"build_ext": BuildExt, "sdist": Sdist}, @@ -126,7 +126,7 @@ setup( packages=["msgpack"], description=desc, long_description=long_desc, - long_description_content_type="text/x-rst", + long_description_content_type="text/markdown", url="https://msgpack.org/", project_urls={ "Documentation": "https://msgpack-python.readthedocs.io/", From a05fc5e7c543d4d925802b6bba1a5542ee8ee3c3 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 12 Dec 2019 18:46:55 +0900 Subject: [PATCH 1029/1172] black --- msgpack/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index 5762e8c..56f67e4 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (1, 0, 0, 'rc1') +version = (1, 0, 0, "rc1") From aab29ff277cf88ff85e7ea5e603607a24d8c38a4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 12 Dec 2019 18:48:16 +0900 Subject: [PATCH 1030/1172] Remove TRANSITIONAL package support --- setup.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/setup.py b/setup.py index ac0dc30..2ec9ca7 100755 --- a/setup.py +++ b/setup.py @@ -14,10 +14,6 @@ PYPY = hasattr(sys, "pypy_version_info") PY2 = sys.version_info[0] == 2 -# for building transitional package. -TRANSITIONAL = False - - class NoCython(Exception): pass @@ -110,14 +106,8 @@ with io.open("README.md", encoding="utf-8") as f: long_desc = f.read() del f -name = "msgpack" - -if TRANSITIONAL: - name = "msgpack-python" - long_desc = "This package is deprecated. Install msgpack instead." - setup( - name=name, + name="msgpack", author="Inada Naoki", author_email="songofacandy@gmail.com", version=version_str, From 887d3a7d22865d36d68fdcb5e653ea61d66f0b61 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 12 Dec 2019 19:43:59 +0900 Subject: [PATCH 1031/1172] Refine Timestamp APIs (#395) --- msgpack/ext.py | 99 ++++++++++++++++++++++++------------------ msgpack/fallback.py | 4 +- test/test_timestamp.py | 16 ++++--- 3 files changed, 69 insertions(+), 50 deletions(-) diff --git a/msgpack/ext.py b/msgpack/ext.py index 09adb34..00b759d 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -7,8 +7,11 @@ import struct PY2 = sys.version_info[0] == 2 -if not PY2: - long = int +if PY2: + int_types = (int, long) + _utc = None +else: + int_types = int try: _utc = datetime.timezone.utc except AttributeError: @@ -23,8 +26,6 @@ class ExtType(namedtuple("ExtType", "code data")): raise TypeError("code must be int") if not isinstance(data, bytes): raise TypeError("data must be bytes") - if code == -1: - return Timestamp.from_bytes(data) if not 0 <= code <= 127: raise ValueError("code must be 0~127") return super(ExtType, cls).__new__(cls, code, data) @@ -42,34 +43,26 @@ class Timestamp(object): def __init__(self, seconds, nanoseconds=0): """Initialize a Timestamp object. - :param seconds: Number of seconds since the UNIX epoch (00:00:00 UTC Jan 1 1970, minus leap seconds). May be - negative. If :code:`seconds` includes a fractional part, :code:`nanoseconds` must be 0. - :type seconds: int or float + :param int seconds: + Number of seconds since the UNIX epoch (00:00:00 UTC Jan 1 1970, minus leap seconds). + May be negative. - :param nanoseconds: Number of nanoseconds to add to `seconds` to get fractional time. Maximum is 999_999_999. - Default is 0. - :type nanoseconds: int + :param int nanoseconds: + Number of nanoseconds to add to `seconds` to get fractional time. + Maximum is 999_999_999. Default is 0. Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns. """ - if not isinstance(seconds, (int, long, float)): - raise TypeError("seconds must be numeric") - if not isinstance(nanoseconds, (int, long)): + if not isinstance(seconds, int_types): + raise TypeError("seconds must be an interger") + if not isinstance(nanoseconds, int_types): raise TypeError("nanoseconds must be an integer") - if nanoseconds: - if nanoseconds < 0 or nanoseconds % 1 != 0 or nanoseconds > (1e9 - 1): - raise ValueError( - "nanoseconds must be a non-negative integer less than 999999999." - ) - if not isinstance(seconds, (int, long)): - raise ValueError( - "seconds must be an integer if also providing nanoseconds." - ) - self.nanoseconds = nanoseconds - else: - # round helps with floating point issues - self.nanoseconds = int(round(seconds % 1 * 1e9, 0)) - self.seconds = int(seconds // 1) + if not (0 <= nanoseconds < 10 ** 9): + raise ValueError( + "nanoseconds must be a non-negative integer less than 999999999." + ) + self.seconds = seconds + self.nanoseconds = nanoseconds def __repr__(self): """String representation of Timestamp.""" @@ -137,7 +130,18 @@ class Timestamp(object): data = struct.pack("!Iq", self.nanoseconds, self.seconds) return data - def to_float(self): + @staticmethod + def from_unix(unix_sec): + """Create a Timestamp from posix timestamp in seconds. + + :param unix_float: Posix timestamp in seconds. + :type unix_float: int or float. + """ + seconds = int(unix_sec // 1) + nanoseconds = int((unix_sec % 1) * 10 ** 9) + return Timestamp(seconds, nanoseconds) + + def to_unix(self): """Get the timestamp as a floating-point value. :returns: posix timestamp @@ -146,28 +150,37 @@ class Timestamp(object): return self.seconds + self.nanoseconds / 1e9 @staticmethod - def from_float(unix_float): - seconds = int(unix_float) - nanoseconds = int((unix_float % 1) * 1000000000) - return Timestamp(seconds, nanoseconds) + def from_unix_nano(unix_ns): + """Create a Timestamp from posix timestamp in nanoseconds. - def to_unix_ns(self): + :param int unix_ns: Posix timestamp in nanoseconds. + :rtype: Timestamp + """ + return Timestamp(*divmod(unix_ns, 10 ** 9)) + + def to_unix_nano(self): """Get the timestamp as a unixtime in nanoseconds. :returns: posix timestamp in nanoseconds :rtype: int """ - return int(self.seconds * 1e9 + self.nanoseconds) + return self.seconds * 10 ** 9 + self.nanoseconds - if not PY2: + def to_datetime(self): + """Get the timestamp as a UTC datetime. - def to_datetime(self): - """Get the timestamp as a UTC datetime. + Python 2 is not supported. - :rtype: datetime. - """ - return datetime.datetime.fromtimestamp(self.to_float(), _utc) + :rtype: datetime. + """ + return datetime.datetime.fromtimestamp(self.to_unix(), _utc) - @staticmethod - def from_datetime(dt): - return Timestamp.from_float(dt.timestamp()) + @staticmethod + def from_datetime(dt): + """Create a Timestamp from datetime with tzinfo. + + Python 2 is not supported. + + :rtype: Timestamp + """ + return Timestamp.from_unix(dt.timestamp()) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 9ba98bf..08e8d46 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -691,9 +691,9 @@ class Unpacker(object): if n == -1: # timestamp ts = Timestamp.from_bytes(bytes(obj)) if self._timestamp == 1: - return ts.to_float() + return ts.to_unix() elif self._timestamp == 2: - return ts.to_unix_ns() + return ts.to_unix_nano() elif self._timestamp == 3: return ts.to_datetime() else: diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 822994c..ba5611c 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -37,19 +37,25 @@ def test_timestamp(): assert ts.seconds == 2 ** 63 - 1 and ts.nanoseconds == 999999999 # negative fractional - ts = Timestamp(-2.3) # s: -3, ns: 700000000 + ts = Timestamp.from_unix(-2.3) # s: -3, ns: 700000000 + assert ts.seconds == -3 and ts.nanoseconds == 700000000 assert ts.to_bytes() == b"\x29\xb9\x27\x00\xff\xff\xff\xff\xff\xff\xff\xfd" packed = msgpack.packb(ts) assert packed == b"\xc7\x0c\xff" + ts.to_bytes() unpacked = msgpack.unpackb(packed) assert ts == unpacked - assert ts.seconds == -3 and ts.nanoseconds == 700000000 + + +def test_timestamp_from(): + t = Timestamp(42, 14000) + assert Timestamp.from_unix(42.000014) == t + assert Timestamp.from_unix_nano(42000014000) == t def test_timestamp_to(): - t = Timestamp(42, 14) - assert t.to_float() == 42.000000014 - assert t.to_unix_ns() == 42000000014 + t = Timestamp(42, 14000) + assert t.to_unix() == 42.000014 + assert t.to_unix_nano() == 42000014000 @pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") From 9e5ec95e0292dce8485575310f6b69a618fdbefe Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 12 Dec 2019 19:59:06 +0900 Subject: [PATCH 1032/1172] Make Timestamp hashable (#396) When overriding __eq__, __hash__ should be overridden too. --- msgpack/ext.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/msgpack/ext.py b/msgpack/ext.py index 00b759d..cc34fb2 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -36,6 +36,8 @@ class Timestamp(object): When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. When using pure-Python msgpack, :func:`to_bytes` and :func:`from_bytes` are used to pack and unpack `Timestamp`. + + This class is immutable: Do not override seconds and nanoseconds. """ __slots__ = ["seconds", "nanoseconds"] @@ -78,9 +80,8 @@ class Timestamp(object): ) return False - def __ne__(self, other): - """not-equals method (see :func:`__eq__()`)""" - return not self.__eq__(other) + def __hash__(self): + return hash((self.seconds, self.nanoseconds)) @staticmethod def from_bytes(b): From 5e1fe818e3839c85a38419859bcec6d38979c620 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 12 Dec 2019 20:05:25 +0900 Subject: [PATCH 1033/1172] Reintroduce __ne__ --- msgpack/ext.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/msgpack/ext.py b/msgpack/ext.py index cc34fb2..8341c68 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -80,6 +80,10 @@ class Timestamp(object): ) return False + def __ne__(self, other): + """not-equals method (see :func:`__eq__()`)""" + return not self.__eq__(other) + def __hash__(self): return hash((self.seconds, self.nanoseconds)) From 42f5ecfd514cc5797385df0c72258a16fe645c72 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 13 Dec 2019 15:10:32 +0900 Subject: [PATCH 1034/1172] Fix some typo --- msgpack/fallback.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 08e8d46..263e74e 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -41,7 +41,7 @@ else: if hasattr(sys, "pypy_version_info"): - # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own + # StringIO is slow on PyPy, StringIO is faster. However: PyPy's own # StringBuilder is fastest. from __pypy__ import newlist_hint @@ -147,7 +147,7 @@ def unpackb(packed, **kwargs): if sys.version_info < (2, 7, 6): def _unpack_from(f, b, o=0): - """Explicit typcast for legacy struct.unpack_from""" + """Explicit type cast for legacy struct.unpack_from""" return struct.unpack_from(f, bytes(b), o) @@ -178,7 +178,7 @@ class Unpacker(object): :param int timestamp: Control how timestamp type is unpacked: - 0 - Tiemstamp + 0 - Timestamp 1 - float (Seconds from the EPOCH) 2 - int (Nanoseconds from the EPOCH) 3 - datetime.datetime (UTC). Python 2 is not supported. @@ -749,7 +749,7 @@ class Packer(object): """ MessagePack Packer - usage: + Usage: packer = Packer() astream.write(packer.pack(a)) @@ -783,7 +783,7 @@ class Packer(object): :param bool datetime: If set to true, datetime with tzinfo is packed into Timestamp type. Note that the tzinfo is stripped in the timestamp. - You can get UTC datetime with `timestamp=3` option of the Unapcker. + You can get UTC datetime with `timestamp=3` option of the Unpacker. (Python 2 is not supported). :param str unicode_errors: From ebfe55e63738fa41e55ac986ccae5e5f9bc3afbd Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 16 Dec 2019 15:14:34 +0900 Subject: [PATCH 1035/1172] travis: Use build config validation. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 39db1d7..7af7beb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +version: ~> 1.0 dist: xenial language: python cache: pip From 030bb2f1f7ebd12faed937d8ef9846601361fbce Mon Sep 17 00:00:00 2001 From: ossdev07 <39188636+ossdev07@users.noreply.github.com> Date: Tue, 31 Dec 2019 06:42:21 +0530 Subject: [PATCH 1036/1172] travis: Add test for arm64 (#399) Signed-off-by: ossdev07 --- .travis.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7af7beb..852674d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,9 @@ version: ~> 1.0 dist: xenial language: python cache: pip - +arch: + - amd64 + - arm64 python: # Available Python (PyPy) can be listed by: # @@ -38,6 +40,21 @@ matrix: script: - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh + - arch: arm64 + name: arm64 32bit build + language: python + services: + - docker + env: + - DOCKER_IMAGE=quay.io/pypa/manylinux2014_aarch64 + install: + - pip install -U pip + - pip install -r requirements.txt + - make cython + - docker pull $DOCKER_IMAGE + script: + - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh + - name: "Python 2 (fallback)" python: "2.7" <<: *pure From 1bd6fc36d09cea273a5b47d4f54ed5e3b718582c Mon Sep 17 00:00:00 2001 From: Emilio Tagua <53828441+eb-emilio@users.noreply.github.com> Date: Wed, 5 Feb 2020 09:20:17 -0300 Subject: [PATCH 1037/1172] Update README.md (#402) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 897a932..48401be 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ See note below for detail. * ``raw=False`` by default. It assumes str types are valid UTF-8 string and decode them to Python str (unicode) object. - * ``encdoding`` option is rmeoved. You can use ``raw=True`` to support old format. + * ``encoding`` option is removed. You can use ``raw=True`` to support old format. * Default value of ``max_buffer_size`` is changed from 0 to 100 MiB. * Default value of ``strict_map_key`` is changed to True to avoid hashdos. You need to pass ``strict_map_key=False`` if you have data which contain map keys From 24950990f4ebeffbf98acd188b171cc60a27095e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 6 Feb 2020 20:29:33 +0900 Subject: [PATCH 1038/1172] Remove broken example --- README.md | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/README.md b/README.md index 48401be..a4d69c1 100644 --- a/README.md +++ b/README.md @@ -214,24 +214,6 @@ the result, or ignoring it. The latter two methods return the number of elements in the upcoming container, so that each element in an array, or key-value pair in a map, can be unpacked or skipped individually. -Each of these methods may optionally write the packed data it reads to a -callback function: - -```py - from io import BytesIO - - def distribute(unpacker, get_worker): - nelems = unpacker.read_map_header() - for i in range(nelems): - # Select a worker for the given key - key = unpacker.unpack() - worker = get_worker(key) - - # Send the value as a packed message to worker - bytestream = BytesIO() - unpacker.skip(bytestream.write) - worker.send(bytestream.getvalue()) -``` ## Notes From 0dad82116912878dfb172de3f5affe128c7475ce Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 6 Feb 2020 20:35:41 +0900 Subject: [PATCH 1039/1172] Fix markdown --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a4d69c1..46af810 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ## What's this -`MessagePack `_ is an efficient binary serialization format. +[MessagePack](https://msgpack.org/) is an efficient binary serialization format. It lets you exchange data among multiple languages like JSON. But it's faster and smaller. This package provides CPython bindings for reading and writing MessagePack data. From ff1f5f89d997b40e60472c2820ea55cab752c779 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 6 Feb 2020 21:06:04 +0900 Subject: [PATCH 1040/1172] README: `` -> ` --- README.md | 74 +++++++++++++++++++++++++++---------------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 46af810..0355286 100644 --- a/README.md +++ b/README.md @@ -28,10 +28,10 @@ msgpack is removed, and `import msgpack` fail. ### Compatibility with the old format -You can use ``use_bin_type=False`` option to pack ``bytes`` +You can use `use_bin_type=False` option to pack `bytes` object into raw type in the old msgpack spec, instead of bin type in new msgpack spec. -You can unpack old msgpack format using ``raw=True`` option. +You can unpack old msgpack format using `raw=True` option. It unpacks str (raw) type in msgpack into Python bytes. See note below for detail. @@ -42,23 +42,23 @@ See note below for detail. * Python 2 * The extension module does not support Python 2 anymore. - The pure Python implementation (``msgpack.fallback``) is used for Python 2. + The pure Python implementation (`msgpack.fallback`) is used for Python 2. * Packer - * ``use_bin_type=True`` by default. bytes are encoded in bin type in msgpack. + * `use_bin_type=True` by default. bytes are encoded in bin type in msgpack. **If you are still sing Python 2, you must use unicode for all string types.** - You can use ``use_bin_type=False`` to encode into old msgpack format. - * ``encoding`` option is removed. UTF-8 is used always. + You can use `use_bin_type=False` to encode into old msgpack format. + * `encoding` option is removed. UTF-8 is used always. * Unpacker - * ``raw=False`` by default. It assumes str types are valid UTF-8 string + * `raw=False` by default. It assumes str types are valid UTF-8 string and decode them to Python str (unicode) object. - * ``encoding`` option is removed. You can use ``raw=True`` to support old format. - * Default value of ``max_buffer_size`` is changed from 0 to 100 MiB. - * Default value of ``strict_map_key`` is changed to True to avoid hashdos. - You need to pass ``strict_map_key=False`` if you have data which contain map keys + * `encoding` option is removed. You can use `raw=True` to support old format. + * Default value of `max_buffer_size` is changed from 0 to 100 MiB. + * Default value of `strict_map_key` is changed to True to avoid hashdos. + You need to pass `strict_map_key=False` if you have data which contain map keys which type is not bytes or str. @@ -70,10 +70,10 @@ See note below for detail. ### Pure Python implementation -The extension module in msgpack (``msgpack._cmsgpack``) does not support +The extension module in msgpack (`msgpack._cmsgpack`) does not support Python 2 and PyPy. -But msgpack provides a pure Python implementation (``msgpack.fallback``) +But msgpack provides a pure Python implementation (`msgpack.fallback`) for PyPy and Python 2. Since the [pip](https://pip.pypa.io/) uses the pure Python implementation, @@ -89,18 +89,18 @@ Without extension, using pure Python implementation on CPython runs slowly. ## How to use -NOTE: In examples below, I use ``raw=False`` and ``use_bin_type=True`` for users +NOTE: In examples below, I use `raw=False` and `use_bin_type=True` for users using msgpack < 1.0. These options are default from msgpack 1.0 so you can omit them. ### One-shot pack & unpack -Use ``packb`` for packing and ``unpackb`` for unpacking. -msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with -``json`` and ``pickle``. +Use `packb` for packing and `unpackb` for unpacking. +msgpack provides `dumps` and `loads` as an alias for compatibility with +`json` and `pickle`. -``pack`` and ``dump`` packs to a file-like object. -``unpack`` and ``load`` unpacks from a file-like object. +`pack` and `dump` packs to a file-like object. +`unpack` and `load` unpacks from a file-like object. ```pycon >>> import msgpack @@ -110,14 +110,14 @@ msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with [1, 2, 3] ``` -``unpack`` unpacks msgpack's array to Python's list, but can also unpack to tuple: +`unpack` unpacks msgpack's array to Python's list, but can also unpack to tuple: ```pycon >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False) (1, 2, 3) ``` -You should always specify the ``use_list`` keyword argument for backward compatibility. +You should always specify the `use_list` keyword argument for backward compatibility. See performance issues relating to `use_list option`_ below. Read the docstring for other options. @@ -125,8 +125,8 @@ Read the docstring for other options. ### Streaming unpacking -``Unpacker`` is a "streaming unpacker". It unpacks multiple objects from one -stream (or from bytes provided through its ``feed`` method). +`Unpacker` is a "streaming unpacker". It unpacks multiple objects from one +stream (or from bytes provided through its `feed` method). ```py import msgpack @@ -147,7 +147,7 @@ stream (or from bytes provided through its ``feed`` method). ### Packing/unpacking of custom data type It is also possible to pack/unpack custom data types. Here is an example for -``datetime.datetime``. +`datetime.datetime`. ```py import datetime @@ -173,8 +173,8 @@ It is also possible to pack/unpack custom data types. Here is an example for this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False) ``` -``Unpacker``'s ``object_hook`` callback receives a dict; the -``object_pairs_hook`` callback may instead be used to receive a list of +`Unpacker`'s `object_hook` callback receives a dict; the +`object_pairs_hook` callback may instead be used to receive a list of key-value pairs. @@ -207,8 +207,8 @@ It is also possible to pack/unpack custom data types using the **ext** type. ### Advanced unpacking control -As an alternative to iteration, ``Unpacker`` objects provide ``unpack``, -``skip``, ``read_array_header`` and ``read_map_header`` methods. The former two +As an alternative to iteration, `Unpacker` objects provide `unpack`, +`skip`, `read_array_header` and `read_map_header` methods. The former two read an entire message from the stream, respectively de-serialising and returning the result, or ignoring it. The latter two methods return the number of elements in the upcoming container, so that each element in an array, or key-value pair @@ -222,8 +222,8 @@ in a map, can be unpacked or skipped individually. Early versions of msgpack didn't distinguish string and binary types. The type for representing both string and binary types was named **raw**. -You can pack into and unpack from this old spec using ``use_bin_type=False`` -and ``raw=True`` options. +You can pack into and unpack from this old spec using `use_bin_type=False` +and `raw=True` options. ```pycon >>> import msgpack @@ -235,7 +235,7 @@ and ``raw=True`` options. ### ext type -To use the **ext** type, pass ``msgpack.ExtType`` object to packer. +To use the **ext** type, pass `msgpack.ExtType` object to packer. ```pycon >>> import msgpack @@ -244,7 +244,7 @@ To use the **ext** type, pass ``msgpack.ExtType`` object to packer. ExtType(code=42, data='xyzzy') ``` -You can use it with ``default`` and ``ext_hook``. See below. +You can use it with `default` and `ext_hook`. See below. ### Security @@ -252,24 +252,24 @@ You can use it with ``default`` and ``ext_hook``. See below. To unpacking data received from unreliable source, msgpack provides two security options. -``max_buffer_size`` (default: 100*1024*1024) limits the internal buffer size. +`max_buffer_size` (default: `100*1024*1024`) limits the internal buffer size. It is used to limit the preallocated list size too. -``strict_map_key`` (default: ``True``) limits the type of map keys to bytes and str. +`strict_map_key` (default: `True`) limits the type of map keys to bytes and str. While msgpack spec doesn't limit the types of the map keys, there is a risk of the hashdos. -If you need to support other types for map keys, use ``strict_map_key=False``. +If you need to support other types for map keys, use `strict_map_key=False`. ### Performance tips CPython's GC starts when growing allocated object. This means unpacking may cause useless GC. -You can use ``gc.disable()`` when unpacking large message. +You can use `gc.disable()` when unpacking large message. List is the default sequence type of Python. But tuple is lighter than list. -You can use ``use_list=False`` while unpacking when performance is important. +You can use `use_list=False` while unpacking when performance is important. ## Development From 9d79351e99e435b8ca749d57a313441783f67133 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 6 Feb 2020 22:11:04 +0900 Subject: [PATCH 1041/1172] Add some test for timestamp (#403) --- .github/workflows/black.yaml | 2 +- msgpack/fallback.py | 8 +------- test/test_format.py | 19 +++++++------------ test/test_pack.py | 14 +++----------- test/test_stricttype.py | 5 +---- test/test_timestamp.py | 27 +++++++++++++++++++++++++++ 6 files changed, 40 insertions(+), 35 deletions(-) diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index cabd0cc..eda8d07 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -18,4 +18,4 @@ jobs: - name: Black Code Formatter run: | pip install black - black --diff --check msgpack/ test/ + black --diff --check msgpack/ test/ setup.py diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 263e74e..9f6665b 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -77,13 +77,7 @@ else: newlist_hint = lambda size: [] -from .exceptions import ( - BufferFull, - OutOfData, - ExtraData, - FormatError, - StackError, -) +from .exceptions import BufferFull, OutOfData, ExtraData, FormatError, StackError from .ext import ExtType, Timestamp diff --git a/test/test_format.py b/test/test_format.py index d455f7c..fbbc3f9 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -9,29 +9,24 @@ def check(src, should, use_list=0, raw=True): def testSimpleValue(): - check(b"\x93\xc0\xc2\xc3", (None, False, True,)) + check(b"\x93\xc0\xc2\xc3", (None, False, True)) def testFixnum(): - check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", ((0, 64, 127,), (-32, -16, -1,),)) + check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", ((0, 64, 127), (-32, -16, -1))) def testFixArray(): - check( - b"\x92\x90\x91\x91\xc0", ((), ((None,),),), - ) + check(b"\x92\x90\x91\x91\xc0", ((), ((None,),))) def testFixRaw(): - check( - b"\x94\xa0\xa1a\xa2bc\xa3def", (b"", b"a", b"bc", b"def",), - ) + check(b"\x94\xa0\xa1a\xa2bc\xa3def", (b"", b"a", b"bc", b"def")) def testFixMap(): check( - b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", - {False: {None: None}, True: {None: {}}}, + b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", {False: {None: None}, True: {None: {}}} ) @@ -40,7 +35,7 @@ def testUnsignedInt(): b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" b"\xce\xff\xff\xff\xff", - (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,), + (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295), ) @@ -49,7 +44,7 @@ def testSignedInt(): b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" b"\xd2\xff\xff\xff\xff", - (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,), + (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1), ) diff --git a/test/test_pack.py b/test/test_pack.py index 932f760..a51d84c 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -49,7 +49,7 @@ def testPack(): False, (), ((),), - ((), None,), + ((), None), {None: 0}, (1 << 23), ] @@ -69,21 +69,13 @@ def testPackUnicode(): def testPackBytes(): - test_data = [ - b"", - b"abcd", - (b"defgh",), - ] + test_data = [b"", b"abcd", (b"defgh",)] for td in test_data: check(td) def testPackByteArrays(): - test_data = [ - bytearray(b""), - bytearray(b"abcd"), - (bytearray(b"defgh"),), - ] + test_data = [bytearray(b""), bytearray(b"abcd"), (bytearray(b"defgh"),)] for td in test_data: check(td) diff --git a/test/test_stricttype.py b/test/test_stricttype.py index 78e1723..fe9ec6c 100644 --- a/test/test_stricttype.py +++ b/test/test_stricttype.py @@ -22,10 +22,7 @@ def test_tuple(): def default(o): if isinstance(o, tuple): - return { - "__type__": "tuple", - "value": list(o), - } + return {"__type__": "tuple", "value": list(o)} raise TypeError("Unsupported type %s" % (type(o),)) def convert(o): diff --git a/test/test_timestamp.py b/test/test_timestamp.py index ba5611c..823fe04 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -46,6 +46,33 @@ def test_timestamp(): assert ts == unpacked +def test_unpack_timestamp(): + # timestamp 32 + assert msgpack.unpackb(b"\xd6\xff\x00\x00\x00\x00") == Timestamp(0) + + # timestamp 64 + assert msgpack.unpackb(b"\xd7\xff" + b"\x00" * 8) == Timestamp(0) + with pytest.raises(ValueError): + msgpack.unpackb(b"\xd7\xff" + b"\xff" * 8) + + # timestamp 96 + assert msgpack.unpackb(b"\xc7\x0c\xff" + b"\x00" * 12) == Timestamp(0) + with pytest.raises(ValueError): + msgpack.unpackb(b"\xc7\x0c\xff" + b"\xff" * 12) == Timestamp(0) + + # Undefined + with pytest.raises(ValueError): + msgpack.unpackb(b"\xd4\xff\x00") # fixext 1 + with pytest.raises(ValueError): + msgpack.unpackb(b"\xd5\xff\x00\x00") # fixext 2 + with pytest.raises(ValueError): + msgpack.unpackb(b"\xc7\x00\xff") # ext8 (len=0) + with pytest.raises(ValueError): + msgpack.unpackb(b"\xc7\x03\xff\0\0\0") # ext8 (len=3) + with pytest.raises(ValueError): + msgpack.unpackb(b"\xc7\x05\xff\0\0\0\0\0") # ext8 (len=5) + + def test_timestamp_from(): t = Timestamp(42, 14000) assert Timestamp.from_unix(42.000014) == t From f0952f1dd657e3f8437907bfe13885e61d5367fe Mon Sep 17 00:00:00 2001 From: Erik Cederstrand Date: Fri, 14 Feb 2020 04:31:09 +0100 Subject: [PATCH 1042/1172] travis: Python 3.9 is the new dev version. (#405) --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 852674d..5132b4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,8 @@ python: - "3.5" - "3.6" - "3.7" - - "3.8-dev" + - "3.8" + - "3.9-dev" _pure: &pure From cd6561db520de20c681bb50022cec621c0deda82 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 14 Feb 2020 06:45:17 +0000 Subject: [PATCH 1043/1172] build: Don't test C extension on CPython 2.7 under Tox (#406) As the Changelog notes, release 1.0 will drop support for the native extension on CPython 2.x. So there seems little benefit of testing it. --- tox.ini | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4b059ff..607b182 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,10 @@ [tox] -envlist = {py27,py35,py36,py37,py38}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86 +envlist = + py27-pure, + {py35,py36,py37,py38}-{c,pure}, + {pypy,pypy3}-pure, + py27-x86, + py34-x86, [variants:pure] setenv= From fcb19a0e1a86d80c28447b0008504c3f4e2faf59 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 14 Feb 2020 06:51:19 +0000 Subject: [PATCH 1044/1172] Clean msgpack/_cmsgpack.cpp and msgpack/_cmsgpack.*.so (#407) --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e2f25cf..b4749b3 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,8 @@ serve-doc: all .PHONY: clean clean: rm -rf build - rm -f msgpack/_msgpack.cpp + rm -f msgpack/_cmsgpack.cpp + rm -f msgpack/_cmsgpack.*.so rm -rf msgpack/__pycache__ rm -rf test/__pycache__ From 64f59884a1a56a882ae888af354de12d5eb052a8 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 17 Feb 2020 16:58:25 +0900 Subject: [PATCH 1045/1172] Add note --- DEVELOP.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 DEVELOP.md diff --git a/DEVELOP.md b/DEVELOP.md new file mode 100644 index 0000000..0e6e0e5 --- /dev/null +++ b/DEVELOP.md @@ -0,0 +1,9 @@ +# Developer's note + +## Wheels + +Wheels for macOS and Linux are built on Travis and AppVeyr, in +[methane/msgpack-wheels](https://github.com/methane/msgpack-wheels) repository. + +Wheels for Windows are built on Github Actions in this repository. + From fa7d7447fc2cc7cc1a8b388618549c98d1712b9c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 17 Feb 2020 17:07:18 +0900 Subject: [PATCH 1046/1172] 1.0.0 --- ChangeLog.rst | 2 +- msgpack/_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index c70b966..55cfd5d 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,7 +1,7 @@ 1.0.0 ===== -Release Date: TBD +Release Date: 2020-02-17 * Remove Python 2 support from the ``msgpack/_cmsgpack``. ``msgpack/fallback`` still supports Python 2. diff --git a/msgpack/_version.py b/msgpack/_version.py index 56f67e4..9f55cf5 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (1, 0, 0, "rc1") +version = (1, 0, 0) From 12506d8d91ce5a7b76c2d8babe7e2d1e2851d3a2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 17 Feb 2020 17:12:47 +0900 Subject: [PATCH 1047/1172] update README --- DEVELOP.md | 16 ++++++++++++++++ README.md | 12 ------------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/DEVELOP.md b/DEVELOP.md index 0e6e0e5..9c823c3 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -7,3 +7,19 @@ Wheels for macOS and Linux are built on Travis and AppVeyr, in Wheels for Windows are built on Github Actions in this repository. + +### Build + +``` +$ make cython +``` + + +### Test + +MessagePack uses `pytest` for testing. +Run test with following command: + +``` +$ make test +``` diff --git a/README.md b/README.md index 0355286..78bc0cf 100644 --- a/README.md +++ b/README.md @@ -270,15 +270,3 @@ You can use `gc.disable()` when unpacking large message. List is the default sequence type of Python. But tuple is lighter than list. You can use `use_list=False` while unpacking when performance is important. - - -## Development - -### Test - -MessagePack uses `pytest` for testing. -Run test with following command: - -``` - $ make test -``` From 2849f5582ab154ade7fcd8c23109b1c7bd8d1530 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 19 Feb 2020 00:53:01 +0900 Subject: [PATCH 1048/1172] Build linux and macOS wheels on GitHub Actions. (#409) --- .github/workflows/linux.yml | 67 ++++++++++++++++++++++++++++++++ .github/workflows/mac.yml | 76 +++++++++++++++++++++++++++++++++++++ Makefile | 8 ++-- docker/buildwheel.sh | 8 +++- 4 files changed, 154 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/linux.yml create mode 100644 .github/workflows/mac.yml diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml new file mode 100644 index 0000000..89bdb4e --- /dev/null +++ b/.github/workflows/linux.yml @@ -0,0 +1,67 @@ +name: Build Linux Wheels +on: + push: + pull_request: + create: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v1 + + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Cythonize + shell: bash + run: | + pip install -U pip + pip -V + pip install -r requirements.txt + make cython + #python setup.py sdist + + - name: Build wheels + shell: bash + run: | + make linux-wheel + + - name: Run test (3.8) + run: | + pip install pytest + pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse + pytest -v test + + + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + + - name: Run test (3.7) + run: | + pip install pytest + pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse + pytest -v test + + - name: Set up Python 3.6 + uses: actions/setup-python@v1 + with: + python-version: 3.6 + + - name: Run test (3.6) + run: | + pip install pytest + pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse + pytest -v test + + + - name: Upload Wheels + uses: actions/upload-artifact@v1 + with: + name: linux-wheels + path: ./dist/wheelhouse/ diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml new file mode 100644 index 0000000..fb2c67f --- /dev/null +++ b/.github/workflows/mac.yml @@ -0,0 +1,76 @@ +name: Build macOS Wheels +on: + push: + pull_request: + create: + +jobs: + build: + runs-on: macos-latest + + steps: + - name: Checkout + uses: actions/checkout@v1 + + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: "3.8" + + - name: Cythonize + run: | + pip install -U pip + pip install -r requirements.txt + make cython + + - name: Build wheel + run: | + pip install setuptools wheel + python setup.py bdist_wheel + + - name: Run test + run: | + pip install pytest + pip install -v msgpack --only-binary :all: -f dist/ --no-index + pytest -v test + + + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: "3.7" + + - name: Build wheel + run: | + pip install setuptools wheel + python setup.py bdist_wheel + + - name: Run test + run: | + pip install pytest + pip install -v msgpack --only-binary :all: -f dist/ --no-index + pytest -v test + + + - name: Set up Python 3.6 + uses: actions/setup-python@v1 + with: + python-version: "3.6" + + - name: Build wheel + run: | + pip install setuptools wheel + python setup.py bdist_wheel + + - name: Run test + run: | + pip install pytest + pip install -v msgpack --only-binary :all: -f dist/ --no-index + pytest -v test + + + - name: Upload Wheels + uses: actions/upload-artifact@v1 + with: + name: macos-wheels + path: ./dist/ diff --git a/Makefile b/Makefile index b4749b3..2a4c0af 100644 --- a/Makefile +++ b/Makefile @@ -30,10 +30,10 @@ clean: .PHONY: update-docker update-docker: - docker pull quay.io/pypa/manylinux1_i686 - docker pull quay.io/pypa/manylinux1_x86_64 + docker pull quay.io/pypa/manylinux2010_i686 + docker pull quay.io/pypa/manylinux2010_x86_64 .PHONY: linux-wheel linux-wheel: - docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_i686 bash docker/buildwheel.sh - docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_x86_64 bash docker/buildwheel.sh + docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_i686 bash docker/buildwheel.sh + docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_x86_64 bash docker/buildwheel.sh diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh index c953127..89a2570 100644 --- a/docker/buildwheel.sh +++ b/docker/buildwheel.sh @@ -10,5 +10,11 @@ echo "arch=$ARCH" for V in "${PYTHON_VERSIONS[@]}"; do PYBIN=/opt/python/$V/bin rm -rf build/ # Avoid lib build by narrow Python is used by wide python - $PYBIN/python setup.py bdist_wheel -p manylinux1_${ARCH} + $PYBIN/python setup.py bdist_wheel +done + +cd dist +for whl in *.whl; do + auditwheel repair "$whl" + rm "$whl" done From 2bfc2d0566e24594460078680ad3bd0dc71892ad Mon Sep 17 00:00:00 2001 From: Charles-Axel Dein Date: Mon, 24 Feb 2020 09:51:56 +0100 Subject: [PATCH 1049/1172] Upgrade msgpack if already installed (#414) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 78bc0cf..921f7f8 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ This package provides CPython bindings for reading and writing MessagePack data. ### PyPI package name TL;DR: When upgrading from msgpack-0.4 or earlier, don't do `pip install -U msgpack-python`. -Do `pip uninstall msgpack-python; pip install msgpack` instead. +Do `pip uninstall msgpack-python; pip install -U msgpack` instead. Package name on PyPI was changed to msgpack from 0.5. I upload transitional package (msgpack-python 0.5 which depending on msgpack) From 692e0ee8ff66686dd423aae69b248b67c3bf9ed4 Mon Sep 17 00:00:00 2001 From: Dan Salmon Date: Wed, 18 Mar 2020 00:29:51 +0000 Subject: [PATCH 1050/1172] Fix typo (#416) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 921f7f8..aeeee3a 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ See note below for detail. * Packer * `use_bin_type=True` by default. bytes are encoded in bin type in msgpack. - **If you are still sing Python 2, you must use unicode for all string types.** + **If you are still using Python 2, you must use unicode for all string types.** You can use `use_bin_type=False` to encode into old msgpack format. * `encoding` option is removed. UTF-8 is used always. From 4e10222b5116806864a91fd9f79a70869e0a43c1 Mon Sep 17 00:00:00 2001 From: Kevin Tewouda Date: Wed, 13 May 2020 06:41:15 +0200 Subject: [PATCH 1051/1172] Fix an example in README.md (#423) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aeeee3a..ac52d94 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ It is also possible to pack/unpack custom data types. Here is an example for } def decode_datetime(obj): - if b'__datetime__' in obj: + if '__datetime__' in obj: obj = datetime.datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f") return obj From b04690012d5d77cfe5074893686c4d55ec780300 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 24 May 2020 02:15:04 +0900 Subject: [PATCH 1052/1172] Update doc version Fixes #425 --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 36fa76e..6b432be 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,7 +41,7 @@ master_doc = "index" # General information about the project. project = u"msgpack" -copyright = u"2013, INADA Naoki" +copyright = u"Inada Naoki" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -49,7 +49,7 @@ copyright = u"2013, INADA Naoki" # # The short X.Y version. # The full version, including alpha/beta/rc tags. -version = release = "0.5" +version = release = "1.0" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From c1b1a23f62d5e0ec39a1910d2e9580ce1c13a1cf Mon Sep 17 00:00:00 2001 From: jfolz Date: Mon, 8 Jun 2020 05:14:50 +0200 Subject: [PATCH 1053/1172] Fix Unpacker.tell() (#427) Fixes #426. Co-authored-by: folz --- msgpack/_unpacker.pyx | 10 ++++++++-- msgpack/fallback.py | 15 ++++++++------- test/test_unpack.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 43c93a2..4340e04 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -484,8 +484,10 @@ cdef class Unpacker(object): nread = min(self.buf_tail - self.buf_head, nbytes) ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread) self.buf_head += nread - if len(ret) < nbytes and self.file_like is not None: - ret += self.file_like.read(nbytes - len(ret)) + if nread < nbytes and self.file_like is not None: + ret += self.file_like.read(nbytes - nread) + nread = len(ret) + self.stream_offset += nread return ret def unpack(self): @@ -519,6 +521,10 @@ cdef class Unpacker(object): return self._unpack(read_map_header) def tell(self): + """Returns the current position of the Unpacker in bytes, i.e., the + number of bytes that were read from the input, also the starting + position of the next object. + """ return self.stream_offset def __iter__(self): diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 9f6665b..1e0bbe9 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -365,18 +365,19 @@ class Unpacker(object): return self._buffer[self._buff_i :] def read_bytes(self, n): - ret = self._read(n) + ret = self._read(n, raise_outofdata=False) self._consume() return ret - def _read(self, n): + def _read(self, n, raise_outofdata=True): # (int) -> bytearray - self._reserve(n) + self._reserve(n, raise_outofdata=raise_outofdata) i = self._buff_i - self._buff_i = i + n - return self._buffer[i : i + n] + ret = self._buffer[i : i + n] + self._buff_i = i + len(ret) + return ret - def _reserve(self, n): + def _reserve(self, n, raise_outofdata=True): remain_bytes = len(self._buffer) - self._buff_i - n # Fast path: buffer has n bytes already @@ -404,7 +405,7 @@ class Unpacker(object): self._buffer += read_data remain_bytes -= len(read_data) - if len(self._buffer) < n + self._buff_i: + if len(self._buffer) < n + self._buff_i and raise_outofdata: self._buff_i = 0 # rollback raise OutOfData diff --git a/test/test_unpack.py b/test/test_unpack.py index bc74c4d..057b7bf 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -3,6 +3,11 @@ import sys from msgpack import Unpacker, packb, OutOfData, ExtType from pytest import raises, mark +try: + from itertools import izip as zip +except ImportError: + pass + def test_unpack_array_header_from_file(): f = BytesIO(packb([1, 2, 3, 4])) @@ -64,7 +69,31 @@ def test_unpacker_ext_hook(): assert unpacker.unpack() == {"a": ExtType(2, b"321")} +def test_unpacker_tell(): + objects = 1, 2, u"abc", u"def", u"ghi" + packed = b"\x01\x02\xa3abc\xa3def\xa3ghi" + positions = 1, 2, 6, 10, 14 + unpacker = Unpacker(BytesIO(packed)) + for obj, unp, pos in zip(objects, unpacker, positions): + assert obj == unp + assert pos == unpacker.tell() + + +def test_unpacker_tell_read_bytes(): + objects = 1, u"abc", u"ghi" + packed = b"\x01\x02\xa3abc\xa3def\xa3ghi" + raw_data = b"\x02", b"\xa3def", b"" + lenghts = 1, 4, 999 + positions = 1, 6, 14 + unpacker = Unpacker(BytesIO(packed)) + for obj, unp, pos, n, raw in zip(objects, unpacker, positions, lenghts, raw_data): + assert obj == unp + assert pos == unpacker.tell() + assert unpacker.read_bytes(n) == raw + + if __name__ == "__main__": test_unpack_array_header_from_file() test_unpacker_hook_refcnt() test_unpacker_ext_hook() + test_unpacker_tell() From 3508ca524ebb83b0117a0cc9a08986d933ddb022 Mon Sep 17 00:00:00 2001 From: Contextualist Date: Sun, 21 Jun 2020 22:27:52 -0400 Subject: [PATCH 1054/1172] Fix benchmark extension module import (#428) --- benchmark/benchmark.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 82d0ddb..2e778dd 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -1,7 +1,7 @@ from msgpack import fallback try: - from msgpack import _unpacker, _packer + from msgpack import _cmsgpack has_ext = True except ImportError: @@ -17,14 +17,14 @@ def profile(name, func): def simple(name, data): if has_ext: - packer = _packer.Packer() + packer = _cmsgpack.Packer() profile("packing %s (ext)" % name, lambda: packer.pack(data)) packer = fallback.Packer() profile("packing %s (fallback)" % name, lambda: packer.pack(data)) data = packer.pack(data) if has_ext: - profile("unpacking %s (ext)" % name, lambda: _unpacker.unpackb(data)) + profile("unpacking %s (ext)" % name, lambda: _cmsgpack.unpackb(data)) profile("unpacking %s (fallback)" % name, lambda: fallback.unpackb(data)) From d9ead81021c7b5f034a0475bf9a88e9612cc8e84 Mon Sep 17 00:00:00 2001 From: Markus Gerstel <2102431+Anthchirp@users.noreply.github.com> Date: Fri, 26 Jun 2020 10:15:46 +0100 Subject: [PATCH 1055/1172] Fix a typo in the changelog (#429) --- ChangeLog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 55cfd5d..d922e84 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -6,7 +6,7 @@ Release Date: 2020-02-17 * Remove Python 2 support from the ``msgpack/_cmsgpack``. ``msgpack/fallback`` still supports Python 2. * Remove ``encoding`` option from the Packer and Unpacker. -* Unpacker: The default value of ``max_buffer_type`` is changed to 100MiB. +* Unpacker: The default value of ``max_buffer_size`` is changed to 100MiB. * Unpacker: ``strict_map_key`` is True by default now. * Unpacker: String map keys are interned. * Drop old buffer protocol support. From 5614dd5a898772faa2dfe89bd9f6a5c90c5fafe5 Mon Sep 17 00:00:00 2001 From: Tom Pohl Date: Thu, 23 Jul 2020 10:53:55 +0200 Subject: [PATCH 1056/1172] Allow for timestamps before UNIX epoch (#433) --- msgpack/ext.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/msgpack/ext.py b/msgpack/ext.py index 8341c68..4eb9dd6 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -178,7 +178,9 @@ class Timestamp(object): :rtype: datetime. """ - return datetime.datetime.fromtimestamp(self.to_unix(), _utc) + return datetime.datetime.fromtimestamp(0, _utc) + datetime.timedelta( + seconds=self.to_unix() + ) @staticmethod def from_datetime(dt): From 772c830841a276adb392dd449809764b2826b1f8 Mon Sep 17 00:00:00 2001 From: Peter Fischer Date: Fri, 24 Jul 2020 09:29:15 +0200 Subject: [PATCH 1057/1172] Synchronize handling of datetime in Packer implementations (#434) The handling of datetime is different in the cython and Python implementations. In contrast to the docs, timezone is not required in the Python implementation. --- msgpack/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 1e0bbe9..9739d53 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -921,7 +921,7 @@ class Packer(object): len(obj), dict_iteritems(obj), nest_limit - 1 ) - if self._datetime and check(obj, _DateTime): + if self._datetime and check(obj, _DateTime) and obj.tzinfo is not None: obj = Timestamp.from_datetime(obj) default_used = 1 continue From 8fb709f2e0438862020d8810fa70a81fb5dac7d4 Mon Sep 17 00:00:00 2001 From: Peter Fischer Date: Thu, 30 Jul 2020 16:48:51 +0200 Subject: [PATCH 1058/1172] Fix datetime before epoch on windows in cython implementation (#436) Cython implementation still used datetime.from_timestamp method, which does not work on windows. Update the cython implementation to use utc time and delta and add a regression test to highlight the issue. --- Makefile | 1 + msgpack/unpack.h | 34 +++++++++++++++++++++------------- test/test_timestamp.py | 8 ++++++++ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 2a4c0af..0110ddf 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,7 @@ clean: rm -rf build rm -f msgpack/_cmsgpack.cpp rm -f msgpack/_cmsgpack.*.so + rm -f msgpack/_cmsgpack.*.pyd rm -rf msgpack/__pycache__ rm -rf test/__pycache__ diff --git a/msgpack/unpack.h b/msgpack/unpack.h index debdf71..868b96e 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -341,7 +341,26 @@ static int unpack_callback_ext(unpack_user* u, const char* base, const char* pos else if (u->timestamp == 0) { // Timestamp py = PyObject_CallFunction(u->timestamp_t, "(Lk)", ts.tv_sec, ts.tv_nsec); } - else { // float or datetime + else if (u->timestamp == 3) { // datetime + // Calculate datetime using epoch + delta + // due to limitations PyDateTime_FromTimestamp on Windows with negative timestamps + PyObject *epoch = PyDateTimeAPI->DateTime_FromDateAndTime(1970, 1, 1, 0, 0, 0, 0, u->utc, PyDateTimeAPI->DateTimeType); + if (epoch == NULL) { + return -1; + } + + PyObject* d = PyDelta_FromDSU(0, ts.tv_sec, ts.tv_nsec / 1000); + if (d == NULL) { + Py_DECREF(epoch); + return -1; + } + + py = PyNumber_Add(epoch, d); + + Py_DECREF(epoch); + Py_DECREF(d); + } + else { // float PyObject *a = PyFloat_FromDouble((double)ts.tv_nsec); if (a == NULL) return -1; @@ -358,18 +377,7 @@ static int unpack_callback_ext(unpack_user* u, const char* base, const char* pos a = PyNumber_Add(b, c); Py_DECREF(b); Py_DECREF(c); - - if (u->timestamp == 3) { // datetime - PyObject *t = PyTuple_Pack(2, a, u->utc); - Py_DECREF(a); - if (t == NULL) { - return -1; - } - py = PyDateTime_FromTimestamp(t); - Py_DECREF(t); - } else { // float - py = a; - } + py = a; } } else { py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1); diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 823fe04..edc488a 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -99,6 +99,14 @@ def test_unpack_datetime(): assert unpacked == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc) +@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") +def test_pack_unpack_before_epoch(): + t_in = datetime.datetime(1960, 1, 1, tzinfo=_utc) + packed = msgpack.packb(t_in, datetime=True) + unpacked = msgpack.unpackb(packed, timestamp=3) + assert unpacked == t_in + + @pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_pack_datetime(): t = Timestamp(42, 14000) From 44bc2bd439808ad7563ef8a558ad6ccfe175a66a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 4 Dec 2020 17:23:09 +0900 Subject: [PATCH 1059/1172] Update docstring --- msgpack/_packer.pyx | 8 ++++- msgpack/_unpacker.pyx | 80 +++++++++++++++++-------------------------- msgpack/fallback.py | 25 +++++++++++++- 3 files changed, 63 insertions(+), 50 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index b470646..e6cd2c7 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -63,7 +63,7 @@ cdef class Packer(object): """ MessagePack Packer - usage:: + Usage:: packer = Packer() astream.write(packer.pack(a)) @@ -94,6 +94,12 @@ cdef class Packer(object): This is useful when trying to implement accurate serialization for python types. + :param bool datetime: + If set to true, datetime with tzinfo is packed into Timestamp type. + Note that the tzinfo is stripped in the timestamp. + You can get UTC datetime with `timestamp=3` option of the Unpacker. + (Python 2 is not supported). + :param str unicode_errors: The error handler for encoding unicode. (default: 'strict') DO NOT USE THIS!! This option is kept for very specific usage. diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 4340e04..e4f3f1e 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -212,65 +212,49 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, cdef class Unpacker(object): - """Streaming unpacker. + """ + MessagePack Packer - Arguments: + Usage:: - :param file_like: - File-like object having `.read(n)` method. - If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. + packer = Packer() + astream.write(packer.pack(a)) + astream.write(packer.pack(b)) - :param int read_size: - Used as `file_like.read(read_size)`. (default: `min(1024**2, max_buffer_size)`) + Packer's constructor has some keyword arguments: - :param bool use_list: - If true, unpack msgpack array to Python list. - Otherwise, unpack to Python tuple. (default: True) + :param callable default: + Convert user type to builtin type that Packer supports. + See also simplejson's document. - :param bool raw: - If true, unpack msgpack raw to Python bytes. - Otherwise, unpack to Python str by decoding with UTF-8 encoding (default). + :param bool use_single_float: + Use single precision float type for float. (default: False) - :param bool strict_map_key: - If true (default), only str or bytes are accepted for map (dict) keys. + :param bool autoreset: + Reset buffer after each pack and return its content as `bytes`. (default: True). + If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. - :param callable object_hook: - When specified, it should be callable. - Unpacker calls it with a dict argument after unpacking msgpack map. - (See also simplejson) + :param bool use_bin_type: + Use bin type introduced in msgpack spec 2.0 for bytes. + It also enables str8 type for unicode. (default: True) - :param callable object_pairs_hook: - When specified, it should be callable. - Unpacker calls it with a list of key-value pairs after unpacking msgpack map. - (See also simplejson) + :param bool strict_types: + If set to true, types will be checked to be exact. Derived classes + from serializable types will not be serialized and will be + treated as unsupported type and forwarded to default. + Additionally tuples will not be serialized as lists. + This is useful when trying to implement accurate serialization + for python types. - :param int max_buffer_size: - Limits size of data waiting unpacked. 0 means system's INT_MAX. - The default value is 100*1024*1024 (100MiB). - Raises `BufferFull` exception when it is insufficient. - You should set this parameter when unpacking data from untrusted source. - - :param int max_str_len: - Deprecated, use *max_buffer_size* instead. - Limits max length of str. (default: max_buffer_size) - - :param int max_bin_len: - Deprecated, use *max_buffer_size* instead. - Limits max length of bin. (default: max_buffer_size) - - :param int max_array_len: - Limits max length of array. (default: max_buffer_size) - - :param int max_map_len: - Limits max length of map. (default: max_buffer_size//2) - - :param int max_ext_len: - Deprecated, use *max_buffer_size* instead. - Limits max size of ext type. (default: max_buffer_size) + :param bool datetime: + If set to true, datetime with tzinfo is packed into Timestamp type. + Note that the tzinfo is stripped in the timestamp. + You can get UTC datetime with `timestamp=3` option of the Unpacker. + (Python 2 is not supported). :param str unicode_errors: - Error handler used for decoding str type. (default: `'strict'`) - + The error handler for encoding unicode. (default: 'strict') + DO NOT USE THIS!! This option is kept for very specific usage. Example of streaming deserialize from file-like object:: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 9739d53..0bfa94e 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -744,7 +744,7 @@ class Packer(object): """ MessagePack Packer - Usage: + Usage:: packer = Packer() astream.write(packer.pack(a)) @@ -784,6 +784,29 @@ class Packer(object): :param str unicode_errors: The error handler for encoding unicode. (default: 'strict') DO NOT USE THIS!! This option is kept for very specific usage. + + Example of streaming deserialize from file-like object:: + + unpacker = Unpacker(file_like) + for o in unpacker: + process(o) + + Example of streaming deserialize from socket:: + + unpacker = Unpacker() + while True: + buf = sock.recv(1024**2) + if not buf: + break + unpacker.feed(buf) + for o in unpacker: + process(o) + + Raises ``ExtraData`` when *packed* contains extra bytes. + Raises ``OutOfData`` when *packed* is incomplete. + Raises ``FormatError`` when *packed* is not valid msgpack. + Raises ``StackError`` when *packed* contains too nested. + Other exceptions can be raised during unpacking. """ def __init__( From 2df517999b012737b1cf36f12bac13541a8eb6e1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 11 Dec 2020 13:39:24 +0900 Subject: [PATCH 1060/1172] Travis: Reduce build Save credits. --- .travis.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5132b4c..5980d63 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,20 +3,17 @@ dist: xenial language: python cache: pip arch: - - amd64 - arm64 + python: # Available Python (PyPy) can be listed by: # # $ aws s3 ls s3://travis-python-archives/binaries/ubuntu/16.04/x86_64/ - - "3.4" - - "3.5" - "3.6" - "3.7" - "3.8" - "3.9-dev" - _pure: &pure install: - pip install -U pip From 7d6b4dfb516451735150d3f796f0b35ed5f004a4 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg <65945052+tsahee@users.noreply.github.com> Date: Fri, 11 Dec 2020 07:30:49 +0200 Subject: [PATCH 1061/1172] Build arm64 wheels (#439) --- .github/workflows/linux.yml | 10 ++++++++++ Makefile | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 89bdb4e..50a0abe 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -30,6 +30,16 @@ jobs: run: | make linux-wheel + - name: Install qemu-user-static for docker + shell: bash + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Build arm64 wheels + shell: bash + run: | + make linux-arm64-wheel + - name: Run test (3.8) run: | pip install pytest diff --git a/Makefile b/Makefile index 0110ddf..05cca55 100644 --- a/Makefile +++ b/Makefile @@ -38,3 +38,7 @@ update-docker: linux-wheel: docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_i686 bash docker/buildwheel.sh docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_x86_64 bash docker/buildwheel.sh + +.PHONY: linux-arm64-wheel +linux-arm64-wheel: + docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2014_aarch64 bash docker/buildwheel.sh From d893697eab07a2cf2a02f5115d4e7bae99d07e3f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 11 Dec 2020 19:16:14 +0900 Subject: [PATCH 1062/1172] v1.0.1 --- ChangeLog.rst | 8 ++++++++ msgpack/_version.py | 2 +- setup.py | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index d922e84..bb3a633 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,11 @@ +1.0.1 +===== + +* Add Python 3.9 and linux/arm64 wheels. (#439) +* Fixed Unpacker.tell() after read_bytes() (#426) +* Fixed unpacking datetime before epoch on Windows (#433) +* Fixed fallback Packer didn't check DateTime.tzinfo (#434) + 1.0.0 ===== diff --git a/msgpack/_version.py b/msgpack/_version.py index 9f55cf5..95e8129 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (1, 0, 0) +version = (1, 0, 1) diff --git a/setup.py b/setup.py index 2ec9ca7..8e88750 100755 --- a/setup.py +++ b/setup.py @@ -132,6 +132,7 @@ setup( "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Intended Audience :: Developers", From edd56036616e5f7211d5cd840a4d00bb43a7f9ee Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 11 Dec 2020 19:31:24 +0900 Subject: [PATCH 1063/1172] Actions: Add Python 3.9 --- .github/workflows/linux.yml | 11 +++++++++++ .github/workflows/mac.yml | 17 +++++++++++++++++ .github/workflows/windows.yaml | 14 ++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 50a0abe..f9707b1 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -47,6 +47,17 @@ jobs: pytest -v test + - name: Set up Python 3.9 + uses: actions/setup-python@v1 + with: + python-version: 3.9 + + - name: Run test (3.9) + run: | + pip install pytest + pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse + pytest -v test + - name: Set up Python 3.7 uses: actions/setup-python@v1 with: diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index fb2c67f..78d944c 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -35,6 +35,23 @@ jobs: pytest -v test + - name: Set up Python 3.9 + uses: actions/setup-python@v1 + with: + python-version: "3.9" + + - name: Build wheel + run: | + pip install setuptools wheel + python setup.py bdist_wheel + + - name: Run test + run: | + pip install pytest + pip install -v msgpack --only-binary :all: -f dist/ --no-index + pytest -v test + + - name: Set up Python 3.7 uses: actions/setup-python@v1 with: diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml index cecb825..139a5a6 100644 --- a/.github/workflows/windows.yaml +++ b/.github/workflows/windows.yaml @@ -63,6 +63,20 @@ jobs: run: | ci/runtests.sh + - name: Python 3.9 (amd64) + env: + PYTHON: "py -3.9-64" + shell: bash + run: | + ci/runtests.sh + + - name: Python 3.9 (x86) + env: + PYTHON: "py -3.9-32" + shell: bash + run: | + ci/runtests.sh + - name: Upload Wheels uses: actions/upload-artifact@v1 with: From 8029f95516dbfddf2fea61efb06dc08ded84aab7 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 11 Dec 2020 19:33:20 +0900 Subject: [PATCH 1064/1172] Add Python 3.9 wheels --- docker/shared.env | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/shared.env b/docker/shared.env index 17abdd8..3601a07 100644 --- a/docker/shared.env +++ b/docker/shared.env @@ -1,4 +1,5 @@ PYTHON_VERSIONS=( + cp39-cp39 cp38-cp38 cp37-cp37m cp36-cp36m From 753b3706d80a7bc5a29147730804e867b97eee57 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 18 Dec 2020 14:21:27 +0900 Subject: [PATCH 1065/1172] Fix overflow in unpacking timestamp to datetime (#452) --- msgpack/unpack.h | 6 +++--- test/test_timestamp.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 868b96e..34212bc 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -342,21 +342,21 @@ static int unpack_callback_ext(unpack_user* u, const char* base, const char* pos py = PyObject_CallFunction(u->timestamp_t, "(Lk)", ts.tv_sec, ts.tv_nsec); } else if (u->timestamp == 3) { // datetime - // Calculate datetime using epoch + delta + // Calculate datetime using epoch + delta // due to limitations PyDateTime_FromTimestamp on Windows with negative timestamps PyObject *epoch = PyDateTimeAPI->DateTime_FromDateAndTime(1970, 1, 1, 0, 0, 0, 0, u->utc, PyDateTimeAPI->DateTimeType); if (epoch == NULL) { return -1; } - PyObject* d = PyDelta_FromDSU(0, ts.tv_sec, ts.tv_nsec / 1000); + PyObject* d = PyDelta_FromDSU(ts.tv_sec/(24*3600), ts.tv_sec%(24*3600), ts.tv_nsec / 1000); if (d == NULL) { Py_DECREF(epoch); return -1; } py = PyNumber_Add(epoch, d); - + Py_DECREF(epoch); Py_DECREF(d); } diff --git a/test/test_timestamp.py b/test/test_timestamp.py index edc488a..6a29be7 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -129,3 +129,14 @@ def test_pack_datetime(): assert x assert x[0] == dt assert msgpack.unpackb(packed) is None + + +@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") +def test_issue451(): + # https://github.com/msgpack/msgpack-python/issues/451 + dt = datetime.datetime(2100, 1, 1, 1, 1, tzinfo=_utc) + packed = msgpack.packb(dt, datetime=True) + assert packed == b"\xd6\xff\xf4\x86eL" + + unpacked = msgpack.unpackb(packed, timestamp=3) + assert dt == unpacked From 94336cf914c202718e37f27d664517c9a8c79d50 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 18 Dec 2020 16:03:05 +0900 Subject: [PATCH 1066/1172] Fix some travis builds. (#453) --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5980d63..4974d26 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,7 @@ _pure: &pure matrix: include: - name: 32bit build + arch: amd64 language: python services: - docker @@ -58,10 +59,12 @@ matrix: <<: *pure - name: "pypy2.7" + arch: amd64 python: "pypy2.7-7.1.1" <<: *pure - name: "pypy3" + arch: amd64 python: "pypy3.6-7.1.1" <<: *pure From 051f9ded1fe2343ffdf55e7a5ec2da1dab7638b9 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 18 Dec 2020 16:13:35 +0900 Subject: [PATCH 1067/1172] format markdown --- README.md | 126 +++++++++++++++++++++++++++--------------------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index ac52d94..2d5dd88 100644 --- a/README.md +++ b/README.md @@ -64,9 +64,9 @@ See note below for detail. ## Install - - $ pip install msgpack - +``` +$ pip install msgpack +``` ### Pure Python implementation @@ -103,18 +103,18 @@ msgpack provides `dumps` and `loads` as an alias for compatibility with `unpack` and `load` unpacks from a file-like object. ```pycon - >>> import msgpack - >>> msgpack.packb([1, 2, 3], use_bin_type=True) - '\x93\x01\x02\x03' - >>> msgpack.unpackb(_, raw=False) - [1, 2, 3] +>>> import msgpack +>>> msgpack.packb([1, 2, 3], use_bin_type=True) +'\x93\x01\x02\x03' +>>> msgpack.unpackb(_, raw=False) +[1, 2, 3] ``` `unpack` unpacks msgpack's array to Python's list, but can also unpack to tuple: ```pycon - >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False) - (1, 2, 3) +>>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False) +(1, 2, 3) ``` You should always specify the `use_list` keyword argument for backward compatibility. @@ -129,18 +129,18 @@ Read the docstring for other options. stream (or from bytes provided through its `feed` method). ```py - import msgpack - from io import BytesIO +import msgpack +from io import BytesIO - buf = BytesIO() - for i in range(100): - buf.write(msgpack.packb(i, use_bin_type=True)) +buf = BytesIO() +for i in range(100): + buf.write(msgpack.packb(i, use_bin_type=True)) - buf.seek(0) +buf.seek(0) - unpacker = msgpack.Unpacker(buf, raw=False) - for unpacked in unpacker: - print(unpacked) +unpacker = msgpack.Unpacker(buf, raw=False) +for unpacked in unpacker: + print(unpacked) ``` @@ -150,27 +150,27 @@ It is also possible to pack/unpack custom data types. Here is an example for `datetime.datetime`. ```py - import datetime - import msgpack +import datetime +import msgpack - useful_dict = { - "id": 1, - "created": datetime.datetime.now(), - } +useful_dict = { + "id": 1, + "created": datetime.datetime.now(), +} - def decode_datetime(obj): - if '__datetime__' in obj: - obj = datetime.datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f") - return obj +def decode_datetime(obj): + if '__datetime__' in obj: + obj = datetime.datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f") + return obj - def encode_datetime(obj): - if isinstance(obj, datetime.datetime): - return {'__datetime__': True, 'as_str': obj.strftime("%Y%m%dT%H:%M:%S.%f")} - return obj +def encode_datetime(obj): + if isinstance(obj, datetime.datetime): + return {'__datetime__': True, 'as_str': obj.strftime("%Y%m%dT%H:%M:%S.%f")} + return obj - packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True) - this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False) +packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True) +this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False) ``` `Unpacker`'s `object_hook` callback receives a dict; the @@ -183,25 +183,25 @@ key-value pairs. It is also possible to pack/unpack custom data types using the **ext** type. ```pycon - >>> import msgpack - >>> import array - >>> def default(obj): - ... if isinstance(obj, array.array) and obj.typecode == 'd': - ... return msgpack.ExtType(42, obj.tostring()) - ... raise TypeError("Unknown type: %r" % (obj,)) - ... - >>> def ext_hook(code, data): - ... if code == 42: - ... a = array.array('d') - ... a.fromstring(data) - ... return a - ... return ExtType(code, data) - ... - >>> data = array.array('d', [1.2, 3.4]) - >>> packed = msgpack.packb(data, default=default, use_bin_type=True) - >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False) - >>> data == unpacked - True +>>> import msgpack +>>> import array +>>> def default(obj): +... if isinstance(obj, array.array) and obj.typecode == 'd': +... return msgpack.ExtType(42, obj.tostring()) +... raise TypeError("Unknown type: %r" % (obj,)) +... +>>> def ext_hook(code, data): +... if code == 42: +... a = array.array('d') +... a.fromstring(data) +... return a +... return ExtType(code, data) +... +>>> data = array.array('d', [1.2, 3.4]) +>>> packed = msgpack.packb(data, default=default, use_bin_type=True) +>>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False) +>>> data == unpacked +True ``` @@ -226,11 +226,11 @@ You can pack into and unpack from this old spec using `use_bin_type=False` and `raw=True` options. ```pycon - >>> import msgpack - >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=False), raw=True) - [b'spam', b'eggs'] - >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), raw=False) - [b'spam', 'eggs'] +>>> import msgpack +>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=False), raw=True) +[b'spam', b'eggs'] +>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), raw=False) +[b'spam', 'eggs'] ``` ### ext type @@ -238,10 +238,10 @@ and `raw=True` options. To use the **ext** type, pass `msgpack.ExtType` object to packer. ```pycon - >>> import msgpack - >>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy')) - >>> msgpack.unpackb(packed) - ExtType(code=42, data='xyzzy') +>>> import msgpack +>>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy')) +>>> msgpack.unpackb(packed) +ExtType(code=42, data='xyzzy') ``` You can use it with `default` and `ext_hook`. See below. From f34fca7fb55e2c8adde39c34ac48542649a24d11 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 18 Dec 2020 16:21:41 +0900 Subject: [PATCH 1068/1172] Update readme --- README.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2d5dd88..d8ce9ba 100644 --- a/README.md +++ b/README.md @@ -15,15 +15,10 @@ This package provides CPython bindings for reading and writing MessagePack data. ### PyPI package name -TL;DR: When upgrading from msgpack-0.4 or earlier, don't do `pip install -U msgpack-python`. -Do `pip uninstall msgpack-python; pip install -U msgpack` instead. +Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5. -Package name on PyPI was changed to msgpack from 0.5. -I upload transitional package (msgpack-python 0.5 which depending on msgpack) -for smooth transition from msgpack-python to msgpack. - -Sadly, this doesn't work for upgrade install. After `pip install -U msgpack-python`, -msgpack is removed, and `import msgpack` fail. +When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before +`pip install -U msgpack`. ### Compatibility with the old format From c0516c603f0eb6555117e312f5cdfb383853bc8e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 18 Dec 2020 16:43:04 +0900 Subject: [PATCH 1069/1172] v1.0.2 --- ChangeLog.rst | 5 +++++ msgpack/_version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index bb3a633..230cc30 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,8 @@ +1.0.2 +===== + +* Fix year 2038 problem regression in 1.0.1. (#451) + 1.0.1 ===== diff --git a/msgpack/_version.py b/msgpack/_version.py index 95e8129..1c83c8e 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (1, 0, 1) +version = (1, 0, 2) From 431ef45c8ebff54c2b182dda46e7edb78a4e271b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 18 Dec 2020 17:43:37 +0900 Subject: [PATCH 1070/1172] Use manylinux1 instead of manylinux2010 --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 05cca55..6f29aed 100644 --- a/Makefile +++ b/Makefile @@ -31,13 +31,13 @@ clean: .PHONY: update-docker update-docker: - docker pull quay.io/pypa/manylinux2010_i686 - docker pull quay.io/pypa/manylinux2010_x86_64 + docker pull quay.io/pypa/manylinux1_i686 + docker pull quay.io/pypa/manylinux1_x86_64 .PHONY: linux-wheel linux-wheel: - docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_i686 bash docker/buildwheel.sh - docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_x86_64 bash docker/buildwheel.sh + docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux1_i686 bash docker/buildwheel.sh + docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux1_x86_64 bash docker/buildwheel.sh .PHONY: linux-arm64-wheel linux-arm64-wheel: From 3b71818bb0e4cbafdb58895fa47704563448b6ac Mon Sep 17 00:00:00 2001 From: Guy Tuval Date: Sat, 2 Jan 2021 08:39:37 +0200 Subject: [PATCH 1071/1172] Refactor fallback read header (#441) --- msgpack/fallback.py | 247 +++++++++++++++----------------------------- 1 file changed, 82 insertions(+), 165 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 0bfa94e..7dbc67a 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -1,5 +1,4 @@ """Fallback pure Python implementation of msgpack""" - from datetime import datetime as _DateTime import sys import struct @@ -148,6 +147,38 @@ if sys.version_info < (2, 7, 6): else: _unpack_from = struct.unpack_from +_NO_FORMAT_USED = "" +_MSGPACK_HEADERS = { + 0xC4: (1, _NO_FORMAT_USED, TYPE_BIN), + 0xC5: (2, ">H", TYPE_BIN), + 0xC6: (4, ">I", TYPE_BIN), + 0xC7: (2, "Bb", TYPE_EXT), + 0xC8: (3, ">Hb", TYPE_EXT), + 0xC9: (5, ">Ib", TYPE_EXT), + 0xCA: (4, ">f"), + 0xCB: (8, ">d"), + 0xCC: (1, _NO_FORMAT_USED), + 0xCD: (2, ">H"), + 0xCE: (4, ">I"), + 0xCF: (8, ">Q"), + 0xD0: (1, "b"), + 0xD1: (2, ">h"), + 0xD2: (4, ">i"), + 0xD3: (8, ">q"), + 0xD4: (1, "b1s", TYPE_EXT), + 0xD5: (2, "b2s", TYPE_EXT), + 0xD6: (4, "b4s", TYPE_EXT), + 0xD7: (8, "b8s", TYPE_EXT), + 0xD8: (16, "b16s", TYPE_EXT), + 0xD9: (1, _NO_FORMAT_USED, TYPE_RAW), + 0xDA: (2, ">H", TYPE_RAW), + 0xDB: (4, ">I", TYPE_RAW), + 0xDC: (2, ">H", TYPE_ARRAY), + 0xDD: (4, ">I", TYPE_ARRAY), + 0xDE: (2, ">H", TYPE_MAP), + 0xDF: (4, ">I", TYPE_MAP), +} + class Unpacker(object): """Streaming unpacker. @@ -409,7 +440,7 @@ class Unpacker(object): self._buff_i = 0 # rollback raise OutOfData - def _read_header(self, execute=EX_CONSTRUCT): + def _read_header(self): typ = TYPE_IMMEDIATE n = 0 obj = None @@ -442,187 +473,73 @@ class Unpacker(object): obj = False elif b == 0xC3: obj = True - elif b == 0xC4: - typ = TYPE_BIN - self._reserve(1) - n = self._buffer[self._buff_i] - self._buff_i += 1 + elif 0xC4 <= b <= 0xC6: + size, fmt, typ = _MSGPACK_HEADERS[b] + self._reserve(size) + if len(fmt) > 0: + n = _unpack_from(fmt, self._buffer, self._buff_i)[0] + else: + n = self._buffer[self._buff_i] + self._buff_i += size if n > self._max_bin_len: raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._read(n) - elif b == 0xC5: - typ = TYPE_BIN - self._reserve(2) - n = _unpack_from(">H", self._buffer, self._buff_i)[0] - self._buff_i += 2 - if n > self._max_bin_len: - raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) - obj = self._read(n) - elif b == 0xC6: - typ = TYPE_BIN - self._reserve(4) - n = _unpack_from(">I", self._buffer, self._buff_i)[0] - self._buff_i += 4 - if n > self._max_bin_len: - raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) - obj = self._read(n) - elif b == 0xC7: # ext 8 - typ = TYPE_EXT - self._reserve(2) - L, n = _unpack_from("Bb", self._buffer, self._buff_i) - self._buff_i += 2 + elif 0xC7 <= b <= 0xC9: + size, fmt, typ = _MSGPACK_HEADERS[b] + self._reserve(size) + L, n = _unpack_from(fmt, self._buffer, self._buff_i) + self._buff_i += size if L > self._max_ext_len: raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) - elif b == 0xC8: # ext 16 - typ = TYPE_EXT - self._reserve(3) - L, n = _unpack_from(">Hb", self._buffer, self._buff_i) - self._buff_i += 3 - if L > self._max_ext_len: - raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) - obj = self._read(L) - elif b == 0xC9: # ext 32 - typ = TYPE_EXT - self._reserve(5) - L, n = _unpack_from(">Ib", self._buffer, self._buff_i) - self._buff_i += 5 - if L > self._max_ext_len: - raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) - obj = self._read(L) - elif b == 0xCA: - self._reserve(4) - obj = _unpack_from(">f", self._buffer, self._buff_i)[0] - self._buff_i += 4 - elif b == 0xCB: - self._reserve(8) - obj = _unpack_from(">d", self._buffer, self._buff_i)[0] - self._buff_i += 8 - elif b == 0xCC: - self._reserve(1) - obj = self._buffer[self._buff_i] - self._buff_i += 1 - elif b == 0xCD: - self._reserve(2) - obj = _unpack_from(">H", self._buffer, self._buff_i)[0] - self._buff_i += 2 - elif b == 0xCE: - self._reserve(4) - obj = _unpack_from(">I", self._buffer, self._buff_i)[0] - self._buff_i += 4 - elif b == 0xCF: - self._reserve(8) - obj = _unpack_from(">Q", self._buffer, self._buff_i)[0] - self._buff_i += 8 - elif b == 0xD0: - self._reserve(1) - obj = _unpack_from("b", self._buffer, self._buff_i)[0] - self._buff_i += 1 - elif b == 0xD1: - self._reserve(2) - obj = _unpack_from(">h", self._buffer, self._buff_i)[0] - self._buff_i += 2 - elif b == 0xD2: - self._reserve(4) - obj = _unpack_from(">i", self._buffer, self._buff_i)[0] - self._buff_i += 4 - elif b == 0xD3: - self._reserve(8) - obj = _unpack_from(">q", self._buffer, self._buff_i)[0] - self._buff_i += 8 - elif b == 0xD4: # fixext 1 - typ = TYPE_EXT - if self._max_ext_len < 1: - raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) - self._reserve(2) - n, obj = _unpack_from("b1s", self._buffer, self._buff_i) - self._buff_i += 2 - elif b == 0xD5: # fixext 2 - typ = TYPE_EXT - if self._max_ext_len < 2: - raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) - self._reserve(3) - n, obj = _unpack_from("b2s", self._buffer, self._buff_i) - self._buff_i += 3 - elif b == 0xD6: # fixext 4 - typ = TYPE_EXT - if self._max_ext_len < 4: - raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) - self._reserve(5) - n, obj = _unpack_from("b4s", self._buffer, self._buff_i) - self._buff_i += 5 - elif b == 0xD7: # fixext 8 - typ = TYPE_EXT - if self._max_ext_len < 8: - raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) - self._reserve(9) - n, obj = _unpack_from("b8s", self._buffer, self._buff_i) - self._buff_i += 9 - elif b == 0xD8: # fixext 16 - typ = TYPE_EXT - if self._max_ext_len < 16: - raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) - self._reserve(17) - n, obj = _unpack_from("b16s", self._buffer, self._buff_i) - self._buff_i += 17 - elif b == 0xD9: - typ = TYPE_RAW - self._reserve(1) - n = self._buffer[self._buff_i] - self._buff_i += 1 + elif 0xCA <= b <= 0xD3: + size, fmt = _MSGPACK_HEADERS[b] + self._reserve(size) + if len(fmt) > 0: + obj = _unpack_from(fmt, self._buffer, self._buff_i)[0] + else: + obj = self._buffer[self._buff_i] + self._buff_i += size + elif 0xD4 <= b <= 0xD8: + size, fmt, typ = _MSGPACK_HEADERS[b] + if self._max_ext_len < size: + raise ValueError( + "%s exceeds max_ext_len(%s)" % (size, self._max_ext_len) + ) + self._reserve(size + 1) + n, obj = _unpack_from(fmt, self._buffer, self._buff_i) + self._buff_i += size + 1 + elif 0xD9 <= b <= 0xDB: + size, fmt, typ = _MSGPACK_HEADERS[b] + self._reserve(size) + if len(fmt) > 0: + (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + else: + n = self._buffer[self._buff_i] + self._buff_i += size if n > self._max_str_len: raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) - elif b == 0xDA: - typ = TYPE_RAW - self._reserve(2) - (n,) = _unpack_from(">H", self._buffer, self._buff_i) - self._buff_i += 2 - if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._read(n) - elif b == 0xDB: - typ = TYPE_RAW - self._reserve(4) - (n,) = _unpack_from(">I", self._buffer, self._buff_i) - self._buff_i += 4 - if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) - obj = self._read(n) - elif b == 0xDC: - typ = TYPE_ARRAY - self._reserve(2) - (n,) = _unpack_from(">H", self._buffer, self._buff_i) - self._buff_i += 2 + elif 0xDC <= b <= 0xDD: + size, fmt, typ = _MSGPACK_HEADERS[b] + self._reserve(size) + (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + self._buff_i += size if n > self._max_array_len: raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) - elif b == 0xDD: - typ = TYPE_ARRAY - self._reserve(4) - (n,) = _unpack_from(">I", self._buffer, self._buff_i) - self._buff_i += 4 - if n > self._max_array_len: - raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) - elif b == 0xDE: - self._reserve(2) - (n,) = _unpack_from(">H", self._buffer, self._buff_i) - self._buff_i += 2 + elif 0xDE <= b <= 0xDF: + size, fmt, typ = _MSGPACK_HEADERS[b] + self._reserve(size) + (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + self._buff_i += size if n > self._max_map_len: raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) - typ = TYPE_MAP - elif b == 0xDF: - self._reserve(4) - (n,) = _unpack_from(">I", self._buffer, self._buff_i) - self._buff_i += 4 - if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) - typ = TYPE_MAP else: raise FormatError("Unknown header: 0x%x" % b) return typ, n, obj def _unpack(self, execute=EX_CONSTRUCT): - typ, n, obj = self._read_header(execute) + typ, n, obj = self._read_header() if execute == EX_READ_ARRAY_HEADER: if typ != TYPE_ARRAY: From 02e1f7623cd8d0fcd4763d542fc60e2957ee2046 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 27 Jan 2021 01:11:32 +0000 Subject: [PATCH 1072/1172] build: Create tox environments using a known Cython version (#408) This change causes Tox to run the project's setup.py in a virtualenv (default path is .tox/.package). The required version of Cython is installed, rather than whatever version is installed system wide. --- pyproject.toml | 8 ++++++++ requirements.txt | 1 + tox.ini | 1 + 3 files changed, 10 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a9eb8aa --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = [ + # Also declared in requirements.txt, if updating here please also update + # there + "Cython~=0.29.13", + "setuptools >= 35.0.2", +] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index a2cce25..180fe85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ +# Also declared in pyproject.toml, if updating here please also update there Cython~=0.29.13 diff --git a/tox.ini b/tox.ini index 607b182..ace1ba9 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,7 @@ envlist = {pypy,pypy3}-pure, py27-x86, py34-x86, +isolated_build = true [variants:pure] setenv= From cfae52437b0d146f74a14d69cb706712c1fa2c95 Mon Sep 17 00:00:00 2001 From: laike9m Date: Wed, 27 Jan 2021 15:33:14 -0800 Subject: [PATCH 1073/1172] Updated readme about Python 2 support (#456) --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index d8ce9ba..cb81648 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,6 @@ Python 2 and PyPy. But msgpack provides a pure Python implementation (`msgpack.fallback`) for PyPy and Python 2. -Since the [pip](https://pip.pypa.io/) uses the pure Python implementation, -Python 2 support will not be dropped in the foreseeable future. ### Windows From 1e728a2e0b7f263a4c77d6cdb6ec7c7e2a91872f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 12 Feb 2021 16:20:14 +0900 Subject: [PATCH 1074/1172] fix docstring (#459) --- msgpack/_unpacker.pyx | 91 ++++++++++++++++++++++++++++--------------- msgpack/fallback.py | 2 +- 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index e4f3f1e..7bfc3af 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -212,49 +212,76 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, cdef class Unpacker(object): - """ - MessagePack Packer + """Streaming unpacker. - Usage:: + Arguments: - packer = Packer() - astream.write(packer.pack(a)) - astream.write(packer.pack(b)) + :param file_like: + File-like object having `.read(n)` method. + If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. - Packer's constructor has some keyword arguments: + :param int read_size: + Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) - :param callable default: - Convert user type to builtin type that Packer supports. - See also simplejson's document. + :param bool use_list: + If true, unpack msgpack array to Python list. + Otherwise, unpack to Python tuple. (default: True) - :param bool use_single_float: - Use single precision float type for float. (default: False) + :param bool raw: + If true, unpack msgpack raw to Python bytes. + Otherwise, unpack to Python str by decoding with UTF-8 encoding (default). - :param bool autoreset: - Reset buffer after each pack and return its content as `bytes`. (default: True). - If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. + :param int timestamp: + Control how timestamp type is unpacked: - :param bool use_bin_type: - Use bin type introduced in msgpack spec 2.0 for bytes. - It also enables str8 type for unicode. (default: True) + 0 - Timestamp + 1 - float (Seconds from the EPOCH) + 2 - int (Nanoseconds from the EPOCH) + 3 - datetime.datetime (UTC). Python 2 is not supported. - :param bool strict_types: - If set to true, types will be checked to be exact. Derived classes - from serializable types will not be serialized and will be - treated as unsupported type and forwarded to default. - Additionally tuples will not be serialized as lists. - This is useful when trying to implement accurate serialization - for python types. + :param bool strict_map_key: + If true (default), only str or bytes are accepted for map (dict) keys. - :param bool datetime: - If set to true, datetime with tzinfo is packed into Timestamp type. - Note that the tzinfo is stripped in the timestamp. - You can get UTC datetime with `timestamp=3` option of the Unpacker. - (Python 2 is not supported). + :param callable object_hook: + When specified, it should be callable. + Unpacker calls it with a dict argument after unpacking msgpack map. + (See also simplejson) + + :param callable object_pairs_hook: + When specified, it should be callable. + Unpacker calls it with a list of key-value pairs after unpacking msgpack map. + (See also simplejson) :param str unicode_errors: - The error handler for encoding unicode. (default: 'strict') - DO NOT USE THIS!! This option is kept for very specific usage. + The error handler for decoding unicode. (default: 'strict') + This option should be used only when you have msgpack data which + contains invalid UTF-8 string. + + :param int max_buffer_size: + Limits size of data waiting unpacked. 0 means 2**32-1. + The default value is 100*1024*1024 (100MiB). + Raises `BufferFull` exception when it is insufficient. + You should set this parameter when unpacking data from untrusted source. + + :param int max_str_len: + Deprecated, use *max_buffer_size* instead. + Limits max length of str. (default: max_buffer_size) + + :param int max_bin_len: + Deprecated, use *max_buffer_size* instead. + Limits max length of bin. (default: max_buffer_size) + + :param int max_array_len: + Limits max length of array. + (default: max_buffer_size) + + :param int max_map_len: + Limits max length of map. + (default: max_buffer_size//2) + + :param int max_ext_len: + Deprecated, use *max_buffer_size* instead. + Limits max size of ext type. (default: max_buffer_size) Example of streaming deserialize from file-like object:: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 7dbc67a..4bcc05d 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -260,7 +260,7 @@ class Unpacker(object): Example of streaming deserialize from socket:: - unpacker = Unpacker(max_buffer_size) + unpacker = Unpacker() while True: buf = sock.recv(1024**2) if not buf: From 4b0819dca941ba3fd27ea127e835698c33705365 Mon Sep 17 00:00:00 2001 From: Andrey Bienkowski Date: Tue, 16 Feb 2021 13:38:06 +0000 Subject: [PATCH 1075/1172] Remove redundant code (#460) --- test/test_unpack.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/test_unpack.py b/test/test_unpack.py index 057b7bf..aa4c01f 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -90,10 +90,3 @@ def test_unpacker_tell_read_bytes(): assert obj == unp assert pos == unpacker.tell() assert unpacker.read_bytes(n) == raw - - -if __name__ == "__main__": - test_unpack_array_header_from_file() - test_unpacker_hook_refcnt() - test_unpacker_ext_hook() - test_unpacker_tell() From 38357b928a2452d0889d80f0a2a721fa66f94c9a Mon Sep 17 00:00:00 2001 From: Andrey Bienkowski Date: Fri, 26 Feb 2021 02:39:36 +0000 Subject: [PATCH 1076/1172] Fix error formatting (#463) --- msgpack/fallback.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 4bcc05d..898fe14 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -455,18 +455,20 @@ class Unpacker(object): n = b & 0b00011111 typ = TYPE_RAW if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len)) obj = self._read(n) elif b & 0b11110000 == 0b10010000: n = b & 0b00001111 typ = TYPE_ARRAY if n > self._max_array_len: - raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) + raise ValueError( + "%s exceeds max_array_len(%s)" % (n, self._max_array_len) + ) elif b & 0b11110000 == 0b10000000: n = b & 0b00001111 typ = TYPE_MAP if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) + raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) elif b == 0xC0: obj = None elif b == 0xC2: @@ -518,7 +520,7 @@ class Unpacker(object): n = self._buffer[self._buff_i] self._buff_i += size if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len)) obj = self._read(n) elif 0xDC <= b <= 0xDD: size, fmt, typ = _MSGPACK_HEADERS[b] @@ -526,14 +528,16 @@ class Unpacker(object): (n,) = _unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_array_len: - raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) + raise ValueError( + "%s exceeds max_array_len(%s)" % (n, self._max_array_len) + ) elif 0xDE <= b <= 0xDF: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) (n,) = _unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) + raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) else: raise FormatError("Unknown header: 0x%x" % b) return typ, n, obj From 4ace82f1087ffa1ca3c44a27c7dd3338739efd0a Mon Sep 17 00:00:00 2001 From: Andrey Bienkowski Date: Fri, 26 Feb 2021 12:08:06 +0000 Subject: [PATCH 1077/1172] Fix tox.ini (#465) There is no such thing as [variants] in the tox syntax. This resulted in MSGPACK_PUREPYTHON being unset in the "pure" test environments --- tox.ini | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index ace1ba9..29c256d 100644 --- a/tox.ini +++ b/tox.ini @@ -7,10 +7,6 @@ envlist = py34-x86, isolated_build = true -[variants:pure] -setenv= - MSGPACK_PUREPYTHON=x - [testenv] deps= pytest @@ -20,6 +16,8 @@ commands= c,x86: python -c 'from msgpack import _cmsgpack' c,x86: py.test pure: py.test +setenv= + pure: MSGPACK_PUREPYTHON=x [testenv:py27-x86] basepython=python2.7-x86 From 44fd5777050c6583791609d3f77e05427bf878a3 Mon Sep 17 00:00:00 2001 From: Alexander Shadchin Date: Sat, 27 Feb 2021 03:30:46 +0300 Subject: [PATCH 1078/1172] Remove unused PyObject_AsReadBuffer definition (#468) Also "old" buffer API was removed in Python 3.10 --- msgpack/_unpacker.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 7bfc3af..9ecfdfb 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -3,7 +3,6 @@ from cpython cimport * cdef extern from "Python.h": ctypedef struct PyObject - cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1 object PyMemoryView_GetContiguous(object obj, int buffertype, char order) from libc.stdlib cimport * From 010de11bedde8d61ced5a382b44a8344d571c305 Mon Sep 17 00:00:00 2001 From: Andrey Bienkowski Date: Sat, 27 Feb 2021 01:50:24 +0000 Subject: [PATCH 1079/1172] Make pure-python wheels and eggs possible (#467) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8e88750..751abff 100755 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ else: macros = [("__LITTLE_ENDIAN__", "1")] ext_modules = [] -if not PYPY and not PY2: +if not PYPY and not PY2 and not os.environ.get("MSGPACK_PUREPYTHON"): ext_modules.append( Extension( "msgpack._cmsgpack", From 38dba9634e4efa7886a777b9e7c739dc148da457 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Thu, 18 Mar 2021 14:35:54 -0700 Subject: [PATCH 1080/1172] cimport uint64_t instead of using ctypedef (#473) --- msgpack/_unpacker.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 9ecfdfb..27facc0 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -8,7 +8,7 @@ cdef extern from "Python.h": from libc.stdlib cimport * from libc.string cimport * from libc.limits cimport * -ctypedef unsigned long long uint64_t +from libc.stdint cimport uint64_t from .exceptions import ( BufferFull, From 09187421eb0ac7061de83c9c112d738aa1edd2cf Mon Sep 17 00:00:00 2001 From: Paul Melis Date: Tue, 16 Nov 2021 06:47:40 +0100 Subject: [PATCH 1081/1172] Improve exception message relating to strict_map_key (#485) --- msgpack/unpack.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 34212bc..23aa622 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -193,7 +193,7 @@ static inline int unpack_callback_map(unpack_user* u, unsigned int n, msgpack_un static inline int unpack_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { if (u->strict_map_key && !PyUnicode_CheckExact(k) && !PyBytes_CheckExact(k)) { - PyErr_Format(PyExc_ValueError, "%.100s is not allowed for map key", Py_TYPE(k)->tp_name); + PyErr_Format(PyExc_ValueError, "%.100s is not allowed for map key when strict_map_key=True", Py_TYPE(k)->tp_name); return -1; } if (PyUnicode_CheckExact(k)) { From 9b84e490e7c78ac9bbd76dcf9ce71c1d0c978d81 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 16 Nov 2021 14:53:08 +0900 Subject: [PATCH 1082/1172] Fix black formatting --- msgpack/fallback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 898fe14..4540875 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -385,7 +385,7 @@ class Unpacker(object): self._buffer.extend(view) def _consume(self): - """ Gets rid of the used parts of the buffer. """ + """Gets rid of the used parts of the buffer.""" self._stream_offset += self._buff_i - self._buf_checkpoint self._buf_checkpoint = self._buff_i From b3f72541925cda3b018942db45a59936184727e3 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 16 Nov 2021 16:19:47 +0900 Subject: [PATCH 1083/1172] Support Python 3.10 and Drop Python 3.5 (#487) * linux: Use manylinux2014 * mac: Drop Python 3.6 too --- .github/workflows/linux.yml | 11 +++++++++++ .github/workflows/mac.yml | 34 +++++++++++++++++----------------- .github/workflows/windows.yaml | 14 ++++++++++++++ Makefile | 4 ++-- docker/buildwheel.sh | 4 +++- docker/shared.env | 2 +- setup.py | 4 +--- 7 files changed, 49 insertions(+), 24 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f9707b1..811bc13 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -47,6 +47,17 @@ jobs: pytest -v test + - name: Set up Python 3.10 + uses: actions/setup-python@v1 + with: + python-version: "3.10" + + - name: Run test (3.10) + run: | + pip install pytest + pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse + pytest -v test + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 78d944c..4efe2ca 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -35,6 +35,23 @@ jobs: pytest -v test + - name: Set up Python 3.10 + uses: actions/setup-python@v1 + with: + python-version: "3.10" + + - name: Build wheel + run: | + pip install setuptools wheel + python setup.py bdist_wheel + + - name: Run test + run: | + pip install pytest + pip install -v msgpack --only-binary :all: -f dist/ --no-index + pytest -v test + + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: @@ -69,23 +86,6 @@ jobs: pytest -v test - - name: Set up Python 3.6 - uses: actions/setup-python@v1 - with: - python-version: "3.6" - - - name: Build wheel - run: | - pip install setuptools wheel - python setup.py bdist_wheel - - - name: Run test - run: | - pip install pytest - pip install -v msgpack --only-binary :all: -f dist/ --no-index - pytest -v test - - - name: Upload Wheels uses: actions/upload-artifact@v1 with: diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml index 139a5a6..debe074 100644 --- a/.github/workflows/windows.yaml +++ b/.github/workflows/windows.yaml @@ -77,6 +77,20 @@ jobs: run: | ci/runtests.sh + - name: Python 3.10 (amd64) + env: + PYTHON: "py -3.10-64" + shell: bash + run: | + ci/runtests.sh + + - name: Python 3.10 (x86) + env: + PYTHON: "py -3.10-32" + shell: bash + run: | + ci/runtests.sh + - name: Upload Wheels uses: actions/upload-artifact@v1 with: diff --git a/Makefile b/Makefile index 6f29aed..b50fa80 100644 --- a/Makefile +++ b/Makefile @@ -36,8 +36,8 @@ update-docker: .PHONY: linux-wheel linux-wheel: - docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux1_i686 bash docker/buildwheel.sh - docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux1_x86_64 bash docker/buildwheel.sh + docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2014_i686 bash docker/buildwheel.sh + docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2014_x86_64 bash docker/buildwheel.sh .PHONY: linux-arm64-wheel linux-arm64-wheel: diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh index 89a2570..ff34139 100644 --- a/docker/buildwheel.sh +++ b/docker/buildwheel.sh @@ -7,10 +7,12 @@ set -e -x ARCH=`uname -p` echo "arch=$ARCH" +ls /opt/python + for V in "${PYTHON_VERSIONS[@]}"; do PYBIN=/opt/python/$V/bin rm -rf build/ # Avoid lib build by narrow Python is used by wide python - $PYBIN/python setup.py bdist_wheel + $PYBIN/python -m build -w done cd dist diff --git a/docker/shared.env b/docker/shared.env index 3601a07..80274ac 100644 --- a/docker/shared.env +++ b/docker/shared.env @@ -1,7 +1,7 @@ PYTHON_VERSIONS=( + cp310-cp310 cp39-cp39 cp38-cp38 cp37-cp37m cp36-cp36m - cp35-cp35m ) diff --git a/setup.py b/setup.py index 751abff..01f125f 100755 --- a/setup.py +++ b/setup.py @@ -125,14 +125,12 @@ setup( }, license="Apache 2.0", classifiers=[ - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Intended Audience :: Developers", From 8e358617e77d63a391f0c0f91b3e552214f2a49a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 16 Nov 2021 17:42:42 +0900 Subject: [PATCH 1084/1172] mac: Provide Universal2 wheel (#488) * mac: Use cibuildwheel * Do not build wheel for PyPy. --- .github/workflows/mac.yml | 51 +++++++++++++++------------------------ 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 4efe2ca..85844e7 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -1,7 +1,6 @@ name: Build macOS Wheels on: push: - pull_request: create: jobs: @@ -12,10 +11,11 @@ jobs: - name: Checkout uses: actions/checkout@v1 - - name: Set up Python 3.8 + # Python 3.9 + - name: Set up Python 3.9 uses: actions/setup-python@v1 with: - python-version: "3.8" + python-version: "3.9" - name: Cythonize run: | @@ -23,66 +23,53 @@ jobs: pip install -r requirements.txt make cython - - name: Build wheel - run: | - pip install setuptools wheel - python setup.py bdist_wheel + - name: Build wheels + uses: pypa/cibuildwheel@v2.2.2 + env: + CIBW_ARCHS_MACOS: x86_64 universal2 + CIBW_SKIP: pp* - name: Run test run: | + ls wheelhouse/ pip install pytest - pip install -v msgpack --only-binary :all: -f dist/ --no-index + pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index pytest -v test - + # Python 3.10 - name: Set up Python 3.10 uses: actions/setup-python@v1 with: python-version: "3.10" - - name: Build wheel - run: | - pip install setuptools wheel - python setup.py bdist_wheel - - name: Run test run: | pip install pytest - pip install -v msgpack --only-binary :all: -f dist/ --no-index + pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index pytest -v test - - - name: Set up Python 3.9 + # Python 3.8 + - name: Set up Python 3.8 uses: actions/setup-python@v1 with: - python-version: "3.9" - - - name: Build wheel - run: | - pip install setuptools wheel - python setup.py bdist_wheel + python-version: "3.8" - name: Run test run: | pip install pytest - pip install -v msgpack --only-binary :all: -f dist/ --no-index + pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index pytest -v test - + # Python 3.7 - name: Set up Python 3.7 uses: actions/setup-python@v1 with: python-version: "3.7" - - name: Build wheel - run: | - pip install setuptools wheel - python setup.py bdist_wheel - - name: Run test run: | pip install pytest - pip install -v msgpack --only-binary :all: -f dist/ --no-index + pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index pytest -v test @@ -90,4 +77,4 @@ jobs: uses: actions/upload-artifact@v1 with: name: macos-wheels - path: ./dist/ + path: ./wheelhouse/ From cfa05d3fdc6290b4847e4781a06ac0668ea9dc18 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 16 Nov 2021 17:47:16 +0900 Subject: [PATCH 1085/1172] Actions: Run CI only for PRs from forks. (#489) --- .github/workflows/black.yaml | 4 ++++ .github/workflows/linux.yml | 4 ++++ .github/workflows/windows.yaml | 7 ++++--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index eda8d07..c545953 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -4,6 +4,10 @@ on: ["push", "pull_request"] jobs: black: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest steps: - name: Setup Python diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 811bc13..95c672b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -6,6 +6,10 @@ on: jobs: build: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml index debe074..0ce50f5 100644 --- a/.github/workflows/windows.yaml +++ b/.github/workflows/windows.yaml @@ -1,14 +1,15 @@ name: Build and test windows wheels on: push: - branches: - - master - - test pull_request: create: jobs: build: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: windows-latest steps: - name: Checkout From e464cb44fa3af5ad3ecd83f9c045b16981d01bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20Egelund-M=C3=BCller?= Date: Tue, 16 Nov 2021 09:49:47 +0100 Subject: [PATCH 1086/1172] Nicer error when packing a datetime without tzinfo (#466) --- msgpack/_packer.pyx | 2 ++ msgpack/fallback.py | 4 ++++ test/test_timestamp.py | 16 ++++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index e6cd2c7..396da0c 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -285,6 +285,8 @@ cdef class Packer(object): o = self._default(o) default_used = 1 continue + elif self.datetime and PyDateTime_CheckExact(o): + PyErr_Format(ValueError, b"can not serialize '%.200s' object where tzinfo=None", Py_TYPE(o).tp_name) else: PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name) return ret diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 4540875..b27acb2 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -874,6 +874,10 @@ class Packer(object): obj = self._default(obj) default_used = 1 continue + + if self._datetime and check(obj, _DateTime): + raise ValueError("Cannot serialize %r where tzinfo=None" % (obj,)) + raise TypeError("Cannot serialize %r" % (obj,)) def pack(self, obj): diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 6a29be7..4e26489 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -140,3 +140,19 @@ def test_issue451(): unpacked = msgpack.unpackb(packed, timestamp=3) assert dt == unpacked + + +@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") +def test_pack_datetime_without_tzinfo(): + dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14) + with pytest.raises(ValueError, match="where tzinfo=None"): + packed = msgpack.packb(dt, datetime=True) + + dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14) + packed = msgpack.packb(dt, datetime=True, default=lambda x: None) + assert packed == msgpack.packb(None) + + dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=_utc) + packed = msgpack.packb(dt, datetime=True) + unpacked = msgpack.unpackb(packed, timestamp=3) + assert unpacked == dt From 724e6200fd6b28b3562e48c73f7827a9c19dd11f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 16 Nov 2021 17:52:01 +0900 Subject: [PATCH 1087/1172] 1.0.3rc1 --- msgpack/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index 1c83c8e..3b462fa 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (1, 0, 2) +version = (1, 0, 3, 'rc1') From e29b423de71cb6da323bce8742b1328603a7f9be Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 17 Nov 2021 11:03:06 +0900 Subject: [PATCH 1088/1172] black --- msgpack/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/_version.py b/msgpack/_version.py index 3b462fa..b716524 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (1, 0, 3, 'rc1') +version = (1, 0, 3, "rc1") From 6129789e9f6ebccdb19b23c2cd1dc9551e57fc53 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 Nov 2021 16:18:17 +0900 Subject: [PATCH 1089/1172] Release v1.0.3 (#491) --- ChangeLog.rst | 9 +++++++++ msgpack/_version.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 230cc30..fc6df68 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,12 @@ +1.0.3 +===== + +Release Date: 2021-11-24 JST + +* Fix Docstring (#459) +* Fix error formatting (#463) +* Improve error message about strict_map_key (#485) + 1.0.2 ===== diff --git a/msgpack/_version.py b/msgpack/_version.py index b716524..fb878b3 100644 --- a/msgpack/_version.py +++ b/msgpack/_version.py @@ -1 +1 @@ -version = (1, 0, 3, "rc1") +version = (1, 0, 3) From bdf0511e29b02427437b03e7d8454bd5076e837f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 Nov 2021 14:43:55 +0900 Subject: [PATCH 1090/1172] Refactor CI (#492) * Use cibuildwheel to build wheels. * Use matrix --- .github/workflows/black.yaml | 2 +- .github/workflows/linux.yml | 103 --------------------------------- .github/workflows/mac.yml | 80 ------------------------- .github/workflows/test.yml | 45 ++++++++++++++ .github/workflows/wheel.yml | 51 ++++++++++++++++ .github/workflows/windows.yaml | 99 ------------------------------- .travis.yml | 89 ---------------------------- Makefile | 5 +- appveyor.yml | 50 ---------------- ci/runtests.bat | 9 --- ci/runtests.sh | 8 --- 11 files changed, 100 insertions(+), 441 deletions(-) delete mode 100644 .github/workflows/linux.yml delete mode 100644 .github/workflows/mac.yml create mode 100644 .github/workflows/test.yml create mode 100644 .github/workflows/wheel.yml delete mode 100644 .github/workflows/windows.yaml delete mode 100644 .travis.yml delete mode 100644 appveyor.yml delete mode 100644 ci/runtests.bat delete mode 100644 ci/runtests.sh diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index c545953..2961ed3 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Setup Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: '3.x' architecture: 'x64' diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml deleted file mode 100644 index 95c672b..0000000 --- a/.github/workflows/linux.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: Build Linux Wheels -on: - push: - pull_request: - create: - -jobs: - build: - # We want to run on external PRs, but not on our own internal PRs as they'll be run - # by the push to the branch. - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - - name: Cythonize - shell: bash - run: | - pip install -U pip - pip -V - pip install -r requirements.txt - make cython - #python setup.py sdist - - - name: Build wheels - shell: bash - run: | - make linux-wheel - - - name: Install qemu-user-static for docker - shell: bash - run: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - - name: Build arm64 wheels - shell: bash - run: | - make linux-arm64-wheel - - - name: Run test (3.8) - run: | - pip install pytest - pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse - pytest -v test - - - - name: Set up Python 3.10 - uses: actions/setup-python@v1 - with: - python-version: "3.10" - - - name: Run test (3.10) - run: | - pip install pytest - pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse - pytest -v test - - - name: Set up Python 3.9 - uses: actions/setup-python@v1 - with: - python-version: 3.9 - - - name: Run test (3.9) - run: | - pip install pytest - pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse - pytest -v test - - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - - name: Run test (3.7) - run: | - pip install pytest - pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse - pytest -v test - - - name: Set up Python 3.6 - uses: actions/setup-python@v1 - with: - python-version: 3.6 - - - name: Run test (3.6) - run: | - pip install pytest - pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse - pytest -v test - - - - name: Upload Wheels - uses: actions/upload-artifact@v1 - with: - name: linux-wheels - path: ./dist/wheelhouse/ diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml deleted file mode 100644 index 85844e7..0000000 --- a/.github/workflows/mac.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Build macOS Wheels -on: - push: - create: - -jobs: - build: - runs-on: macos-latest - - steps: - - name: Checkout - uses: actions/checkout@v1 - - # Python 3.9 - - name: Set up Python 3.9 - uses: actions/setup-python@v1 - with: - python-version: "3.9" - - - name: Cythonize - run: | - pip install -U pip - pip install -r requirements.txt - make cython - - - name: Build wheels - uses: pypa/cibuildwheel@v2.2.2 - env: - CIBW_ARCHS_MACOS: x86_64 universal2 - CIBW_SKIP: pp* - - - name: Run test - run: | - ls wheelhouse/ - pip install pytest - pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index - pytest -v test - - # Python 3.10 - - name: Set up Python 3.10 - uses: actions/setup-python@v1 - with: - python-version: "3.10" - - - name: Run test - run: | - pip install pytest - pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index - pytest -v test - - # Python 3.8 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: "3.8" - - - name: Run test - run: | - pip install pytest - pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index - pytest -v test - - # Python 3.7 - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: "3.7" - - - name: Run test - run: | - pip install pytest - pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index - pytest -v test - - - - name: Upload Wheels - uses: actions/upload-artifact@v1 - with: - name: macos-wheels - path: ./wheelhouse/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..6e497e0 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,45 @@ +name: Run tests +on: + push: + branches: [main] + pull_request: + create: + +jobs: + test: + strategy: + matrix: + os: [ubuntu-20.04, windows-2022, macos-10.15] + py: ["3.10", "3.9", "3.8", "3.7", "3.6"] + + runs-on: ${{ matrix.os }} + name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.py }} + cache: "pip" + + - name: Build + shell: bash + run: | + pip install -U pip + pip install -r requirements.txt pytest + make cython + pip install . + + - name: Test (C extension) + shell: bash + run: | + pytest -v test + + - name: Test (pure Python fallback) + shell: bash + run: | + MSGPACK_PUREPYTHON=1 pytest -v test + diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml new file mode 100644 index 0000000..5627b5f --- /dev/null +++ b/.github/workflows/wheel.yml @@ -0,0 +1,51 @@ +name: Build Wheels +on: + push: + branches: [main] + create: + +jobs: + build_wheels: + strategy: + matrix: + os: [ubuntu-20.04, windows-2022, macos-10.15] + runs-on: ${{ matrix.os }} + name: Build wheels on ${{ matrix.os }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v1 + with: + platforms: arm64 + + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + cache: "pip" + + - name: Prepare + shell: bash + run: | + pip install -r requirements.txt + make cython + + - name: Build + uses: pypa/cibuildwheel@v2.2.2 + env: + CIBW_TEST_REQUIRES: "pytest" + CIBW_TEST_COMMAND: "pytest {package}/test" + CIBW_ARCHS_LINUX: auto aarch64 + CIBW_ARCHS_MACOS: x86_64 universal2 arm64 + CIBW_SKIP: pp* + + - name: Upload Wheels + uses: actions/upload-artifact@v1 + with: + name: Wheels + path: wheelhouse + diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml deleted file mode 100644 index 0ce50f5..0000000 --- a/.github/workflows/windows.yaml +++ /dev/null @@ -1,99 +0,0 @@ -name: Build and test windows wheels -on: - push: - pull_request: - create: - -jobs: - build: - # We want to run on external PRs, but not on our own internal PRs as they'll be run - # by the push to the branch. - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - - runs-on: windows-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - - name: Cythonize - shell: bash - run: | - pip install -U Cython - make cython - #python setup.py sdist - - - name: Python 3.6 (amd64) - env: - PYTHON: "py -3.6-64" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.6 (x86) - env: - PYTHON: "py -3.6-32" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.7 (amd64) - env: - PYTHON: "py -3.7-64" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.7 (x86) - env: - PYTHON: "py -3.7-32" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.8 (amd64) - env: - PYTHON: "py -3.8-64" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.8 (x86) - env: - PYTHON: "py -3.8-32" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.9 (amd64) - env: - PYTHON: "py -3.9-64" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.9 (x86) - env: - PYTHON: "py -3.9-32" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.10 (amd64) - env: - PYTHON: "py -3.10-64" - shell: bash - run: | - ci/runtests.sh - - - name: Python 3.10 (x86) - env: - PYTHON: "py -3.10-32" - shell: bash - run: | - ci/runtests.sh - - - name: Upload Wheels - uses: actions/upload-artifact@v1 - with: - name: win-wheels - path: ./dist diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 4974d26..0000000 --- a/.travis.yml +++ /dev/null @@ -1,89 +0,0 @@ -version: ~> 1.0 -dist: xenial -language: python -cache: pip -arch: - - arm64 - -python: - # Available Python (PyPy) can be listed by: - # - # $ aws s3 ls s3://travis-python-archives/binaries/ubuntu/16.04/x86_64/ - - "3.6" - - "3.7" - - "3.8" - - "3.9-dev" - -_pure: &pure - install: - - pip install -U pip - - pip install -U pytest pytest-cov codecov - - pip install . - script: - - pytest --cov=msgpack -v test - -matrix: - include: - - name: 32bit build - arch: amd64 - language: python - services: - - docker - env: - - DOCKER_IMAGE=quay.io/pypa/manylinux1_i686 - install: - - pip install -U pip - - pip install -r requirements.txt - - make cython - - docker pull $DOCKER_IMAGE - script: - - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh - - - arch: arm64 - name: arm64 32bit build - language: python - services: - - docker - env: - - DOCKER_IMAGE=quay.io/pypa/manylinux2014_aarch64 - install: - - pip install -U pip - - pip install -r requirements.txt - - make cython - - docker pull $DOCKER_IMAGE - script: - - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh - - - name: "Python 2 (fallback)" - python: "2.7" - <<: *pure - - - name: "pypy2.7" - arch: amd64 - python: "pypy2.7-7.1.1" - <<: *pure - - - name: "pypy3" - arch: amd64 - python: "pypy3.6-7.1.1" - <<: *pure - -install: - - pip install -U pip - - pip install -U pytest pytest-cov codecov - - pip install -r requirements.txt # Cython - - make cython - - pip install -e . - -script: - - python -c 'import sys; print(hex(sys.maxsize))' - - python -c 'from msgpack import _cmsgpack' - - pytest --cov=msgpack -v test - - MSGPACK_PUREPYTHON=x pytest --cov=msgpack -v test - -after_success: - - if [ -f .coverage ]; then - codecov; - fi - -# vim: sw=2 ts=2 diff --git a/Makefile b/Makefile index b50fa80..02a4a17 100644 --- a/Makefile +++ b/Makefile @@ -31,8 +31,9 @@ clean: .PHONY: update-docker update-docker: - docker pull quay.io/pypa/manylinux1_i686 - docker pull quay.io/pypa/manylinux1_x86_64 + docker pull quay.io/pypa/manylinux2014_i686 + docker pull quay.io/pypa/manylinux2014_x86_64 + docker pull quay.io/pypa/manylinux2014_aarch64 .PHONY: linux-wheel linux-wheel: diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index f338e17..0000000 --- a/appveyor.yml +++ /dev/null @@ -1,50 +0,0 @@ -environment: - matrix: - # For Python versions available on Appveyor, see - # http://www.appveyor.com/docs/installed-software#python - - PYTHON: "C:\\Python36" - -install: - # We need wheel installed to build wheels - - "%PYTHON%\\python.exe -m pip install -U pip" - - "%PYTHON%\\python.exe -m pip install -U cython" - - "%PYTHON%\\Scripts\\cython --cplus msgpack/_cmsgpack.pyx" - -build: off - -test_script: - # Put your test command here. - # Note that you must use the environment variable %PYTHON% to refer to - # the interpreter you're using - Appveyor does not do anything special - # to put the Python version you want to use on PATH. - - set PYTHON="C:\\Python27" - - ci\\runtests.bat - - set PYTHON="C:\\Python27-x64" - - ci\\runtests.bat - - set PYTHON="C:\\Python36" - - ci\\runtests.bat - - set PYTHON="C:\\Python36-x64" - - ci\\runtests.bat - - set PYTHON="C:\\Python37" - - ci\\runtests.bat - - set PYTHON="C:\\Python37-x64" - - ci\\runtests.bat - - set PYTHON="C:\\Python38" - - ci\\runtests.bat - - set PYTHON="C:\\Python38-x64" - - ci\\runtests.bat - -after_test: - # This step builds your wheels. - # Again, you need to use %PYTHON% to get the correct interpreter - -artifacts: - # bdist_wheel puts your built wheel in the dist directory - - path: dist\*.whl - -#on_success: -# You can use this step to upload your artifacts to a public website. -# See Appveyor's documentation for more details. Or you can simply -# access your wheels from the Appveyor "artifacts" tab for your build. - -# vim: set shiftwidth=2 diff --git a/ci/runtests.bat b/ci/runtests.bat deleted file mode 100644 index 4ae2f70..0000000 --- a/ci/runtests.bat +++ /dev/null @@ -1,9 +0,0 @@ -%PYTHON%\python.exe -m pip install -U pip wheel pytest -%PYTHON%\python.exe setup.py build_ext -i -%PYTHON%\python.exe setup.py install -%PYTHON%\python.exe -c "import sys; print(hex(sys.maxsize))" -%PYTHON%\python.exe -c "from msgpack import _cmsgpack" -%PYTHON%\python.exe setup.py bdist_wheel -%PYTHON%\python.exe -m pytest -v test -SET EL=%ERRORLEVEL% -exit /b %EL% diff --git a/ci/runtests.sh b/ci/runtests.sh deleted file mode 100644 index 5d87f69..0000000 --- a/ci/runtests.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -ex -${PYTHON} -VV -${PYTHON} -m pip install setuptools wheel pytest -${PYTHON} setup.py build_ext -if -${PYTHON} -c "from msgpack import _cmsgpack" -${PYTHON} setup.py bdist_wheel -${PYTHON} -m pytest -v test From 89ea57747ebcb0fad004a92ab00ebf13c10b2d51 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 19 Jan 2022 14:42:28 +0900 Subject: [PATCH 1091/1172] Don't define __*_ENDIAN__ macro on Unix. (#495) --- msgpack/fallback.py | 4 ---- setup.py | 6 ++---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index b27acb2..9731a22 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -11,7 +11,6 @@ if PY2: def dict_iteritems(d): return d.iteritems() - else: int_types = int unicode = str @@ -32,7 +31,6 @@ if sys.version_info < (3, 5): and e.args[0].startswith("maximum recursion depth exceeded") ) - else: def _is_recursionerror(e): @@ -68,7 +66,6 @@ if hasattr(sys, "pypy_version_info"): def getvalue(self): return self.builder.build() - else: USING_STRINGBUILDER = False from io import BytesIO as StringIO @@ -143,7 +140,6 @@ if sys.version_info < (2, 7, 6): """Explicit type cast for legacy struct.unpack_from""" return struct.unpack_from(f, bytes(b), o) - else: _unpack_from = struct.unpack_from diff --git a/setup.py b/setup.py index 01f125f..502ed33 100755 --- a/setup.py +++ b/setup.py @@ -79,12 +79,10 @@ class Sdist(sdist): libraries = [] +macros = [] + if sys.platform == "win32": libraries.append("ws2_32") - -if sys.byteorder == "big": - macros = [("__BIG_ENDIAN__", "1")] -else: macros = [("__LITTLE_ENDIAN__", "1")] ext_modules = [] From cb50b2081b21e5cb4a364d292f55092c98aa1a6f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 3 Mar 2022 12:29:55 +0900 Subject: [PATCH 1092/1172] Update setuptools and black (#498) * Use setuptools * Use black==22.1.0 --- .github/workflows/black.yaml | 6 +++--- Makefile | 2 +- msgpack/__init__.py | 5 ++++- msgpack/_version.py | 1 - msgpack/ext.py | 8 +++---- msgpack/fallback.py | 12 +++++------ requirements.txt | 3 +++ setup.cfg | 32 +++++++++++++++++++++++++++ setup.py | 42 ++---------------------------------- test/test_limits.py | 12 +++++------ test/test_memoryview.py | 28 ++++++++++++------------ test/test_sequnpack.py | 4 ++-- test/test_timestamp.py | 12 +++++------ 13 files changed, 83 insertions(+), 84 deletions(-) delete mode 100644 msgpack/_version.py create mode 100644 setup.cfg diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index 2961ed3..be137ae 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -17,9 +17,9 @@ jobs: architecture: 'x64' - name: Checkout - uses: actions/checkout@v1 + uses: actions/checkout@v2 - name: Black Code Formatter run: | - pip install black - black --diff --check msgpack/ test/ setup.py + pip install black==22.1.0 + black -S --diff --check msgpack/ test/ setup.py diff --git a/Makefile b/Makefile index 02a4a17..415dcfd 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: cython .PHONY: black black: - black msgpack/ test/ setup.py + black -S msgpack/ test/ setup.py .PHONY: cython cython: diff --git a/msgpack/__init__.py b/msgpack/__init__.py index d6705e2..1929df3 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,5 +1,4 @@ # coding: utf-8 -from ._version import version from .exceptions import * from .ext import ExtType, Timestamp @@ -7,6 +6,10 @@ import os import sys +version = (1, 0, 4, 'dev') +__version__ = "1.0.4dev" + + if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: from .fallback import Packer, unpackb, Unpacker else: diff --git a/msgpack/_version.py b/msgpack/_version.py deleted file mode 100644 index fb878b3..0000000 --- a/msgpack/_version.py +++ /dev/null @@ -1 +0,0 @@ -version = (1, 0, 3) diff --git a/msgpack/ext.py b/msgpack/ext.py index 4eb9dd6..25544c5 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -59,7 +59,7 @@ class Timestamp(object): raise TypeError("seconds must be an interger") if not isinstance(nanoseconds, int_types): raise TypeError("nanoseconds must be an integer") - if not (0 <= nanoseconds < 10 ** 9): + if not (0 <= nanoseconds < 10**9): raise ValueError( "nanoseconds must be a non-negative integer less than 999999999." ) @@ -143,7 +143,7 @@ class Timestamp(object): :type unix_float: int or float. """ seconds = int(unix_sec // 1) - nanoseconds = int((unix_sec % 1) * 10 ** 9) + nanoseconds = int((unix_sec % 1) * 10**9) return Timestamp(seconds, nanoseconds) def to_unix(self): @@ -161,7 +161,7 @@ class Timestamp(object): :param int unix_ns: Posix timestamp in nanoseconds. :rtype: Timestamp """ - return Timestamp(*divmod(unix_ns, 10 ** 9)) + return Timestamp(*divmod(unix_ns, 10**9)) def to_unix_nano(self): """Get the timestamp as a unixtime in nanoseconds. @@ -169,7 +169,7 @@ class Timestamp(object): :returns: posix timestamp in nanoseconds :rtype: int """ - return self.seconds * 10 ** 9 + self.nanoseconds + return self.seconds * 10**9 + self.nanoseconds def to_datetime(self): """Get the timestamp as a UTC datetime. diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 9731a22..5f215e9 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -318,7 +318,7 @@ class Unpacker(object): self._buf_checkpoint = 0 if not max_buffer_size: - max_buffer_size = 2 ** 31 - 1 + max_buffer_size = 2**31 - 1 if max_str_len == -1: max_str_len = max_buffer_size if max_bin_len == -1: @@ -800,20 +800,20 @@ class Packer(object): raise OverflowError("Integer value out of range") if check(obj, (bytes, bytearray)): n = len(obj) - if n >= 2 ** 32: + if n >= 2**32: raise ValueError("%s is too large" % type(obj).__name__) self._pack_bin_header(n) return self._buffer.write(obj) if check(obj, unicode): obj = obj.encode("utf-8", self._unicode_errors) n = len(obj) - if n >= 2 ** 32: + if n >= 2**32: raise ValueError("String is too large") self._pack_raw_header(n) return self._buffer.write(obj) if check(obj, memoryview): n = len(obj) * obj.itemsize - if n >= 2 ** 32: + if n >= 2**32: raise ValueError("Memoryview is too large") self._pack_bin_header(n) return self._buffer.write(obj) @@ -895,7 +895,7 @@ class Packer(object): return ret def pack_array_header(self, n): - if n >= 2 ** 32: + if n >= 2**32: raise ValueError self._pack_array_header(n) if self._autoreset: @@ -904,7 +904,7 @@ class Packer(object): return ret def pack_map_header(self, n): - if n >= 2 ** 32: + if n >= 2**32: raise ValueError self._pack_map_header(n) if self._autoreset: diff --git a/requirements.txt b/requirements.txt index 180fe85..9bdb478 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,5 @@ # Also declared in pyproject.toml, if updating here please also update there Cython~=0.29.13 + +# dev only tools. no need to add pyproject +black==22.1.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..1cb6ce3 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,32 @@ +[metadata] +name = msgpack +#version = attr: msgpack.__version__ +version = attr: msgpack.version +license = Apache 2.0 +author = Inada Naoki +author_email = songofacandy@gmail.com +description = MessagePack serializer +long_description = file: README.md +long_description_content_type = text/markdown +url = https://msgpack.org/ + +project_urls = + Documentation = https://msgpack-python.readthedocs.io/ + Source = https://github.com/msgpack/msgpack-python + Tracker = https://github.com/msgpack/msgpack-python/issues + +classifiers = + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: Implementation :: PyPy + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + +[flake8] +max_line_length = 100 + diff --git a/setup.py b/setup.py index 502ed33..9630cda 100755 --- a/setup.py +++ b/setup.py @@ -4,10 +4,9 @@ import io import os import sys from glob import glob -from distutils.command.sdist import sdist from setuptools import setup, Extension - -from distutils.command.build_ext import build_ext +from setuptools.command.build_ext import build_ext +from setuptools.command.sdist import sdist PYPY = hasattr(sys, "pypy_version_info") @@ -65,12 +64,6 @@ class BuildExt(build_ext): print(e) -exec(open("msgpack/_version.py").read()) - -version_str = ".".join(str(x) for x in version[:3]) -if len(version) > 3 and version[3] != "final": - version_str += version[3] - # Cython is required for sdist class Sdist(sdist): def __init__(self, *args, **kwargs): @@ -99,39 +92,8 @@ if not PYPY and not PY2 and not os.environ.get("MSGPACK_PUREPYTHON"): del libraries, macros -desc = "MessagePack (de)serializer." -with io.open("README.md", encoding="utf-8") as f: - long_desc = f.read() -del f - setup( - name="msgpack", - author="Inada Naoki", - author_email="songofacandy@gmail.com", - version=version_str, cmdclass={"build_ext": BuildExt, "sdist": Sdist}, ext_modules=ext_modules, packages=["msgpack"], - description=desc, - long_description=long_desc, - long_description_content_type="text/markdown", - url="https://msgpack.org/", - project_urls={ - "Documentation": "https://msgpack-python.readthedocs.io/", - "Source": "https://github.com/msgpack/msgpack-python", - "Tracker": "https://github.com/msgpack/msgpack-python/issues", - }, - license="Apache 2.0", - classifiers=[ - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - ], ) diff --git a/test/test_limits.py b/test/test_limits.py index 65e6bcc..4314c2c 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -16,12 +16,12 @@ from msgpack import ( def test_integer(): - x = -(2 ** 63) + x = -(2**63) assert unpackb(packb(x)) == x with pytest.raises(PackOverflowError): packb(x - 1) - x = 2 ** 64 - 1 + x = 2**64 - 1 assert unpackb(packb(x)) == x with pytest.raises(PackOverflowError): packb(x + 1) @@ -29,16 +29,16 @@ def test_integer(): def test_array_header(): packer = Packer() - packer.pack_array_header(2 ** 32 - 1) + packer.pack_array_header(2**32 - 1) with pytest.raises(PackValueError): - packer.pack_array_header(2 ** 32) + packer.pack_array_header(2**32) def test_map_header(): packer = Packer() - packer.pack_map_header(2 ** 32 - 1) + packer.pack_map_header(2**32 - 1) with pytest.raises(PackValueError): - packer.pack_array_header(2 ** 32) + packer.pack_array_header(2**32) def test_max_str_len(): diff --git a/test/test_memoryview.py b/test/test_memoryview.py index 86b2c1f..84941db 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -53,46 +53,46 @@ def test_fixstr_from_float(): def test_str16_from_byte(): - _runtest("B", 2 ** 8, b"\xda", b"\x01\x00", False) - _runtest("B", 2 ** 16 - 1, b"\xda", b"\xff\xff", False) + _runtest("B", 2**8, b"\xda", b"\x01\x00", False) + _runtest("B", 2**16 - 1, b"\xda", b"\xff\xff", False) def test_str16_from_float(): - _runtest("f", 2 ** 8, b"\xda", b"\x01\x00", False) - _runtest("f", 2 ** 16 - 4, b"\xda", b"\xff\xfc", False) + _runtest("f", 2**8, b"\xda", b"\x01\x00", False) + _runtest("f", 2**16 - 4, b"\xda", b"\xff\xfc", False) def test_str32_from_byte(): - _runtest("B", 2 ** 16, b"\xdb", b"\x00\x01\x00\x00", False) + _runtest("B", 2**16, b"\xdb", b"\x00\x01\x00\x00", False) def test_str32_from_float(): - _runtest("f", 2 ** 16, b"\xdb", b"\x00\x01\x00\x00", False) + _runtest("f", 2**16, b"\xdb", b"\x00\x01\x00\x00", False) def test_bin8_from_byte(): _runtest("B", 1, b"\xc4", b"\x01", True) - _runtest("B", 2 ** 8 - 1, b"\xc4", b"\xff", True) + _runtest("B", 2**8 - 1, b"\xc4", b"\xff", True) def test_bin8_from_float(): _runtest("f", 4, b"\xc4", b"\x04", True) - _runtest("f", 2 ** 8 - 4, b"\xc4", b"\xfc", True) + _runtest("f", 2**8 - 4, b"\xc4", b"\xfc", True) def test_bin16_from_byte(): - _runtest("B", 2 ** 8, b"\xc5", b"\x01\x00", True) - _runtest("B", 2 ** 16 - 1, b"\xc5", b"\xff\xff", True) + _runtest("B", 2**8, b"\xc5", b"\x01\x00", True) + _runtest("B", 2**16 - 1, b"\xc5", b"\xff\xff", True) def test_bin16_from_float(): - _runtest("f", 2 ** 8, b"\xc5", b"\x01\x00", True) - _runtest("f", 2 ** 16 - 4, b"\xc5", b"\xff\xfc", True) + _runtest("f", 2**8, b"\xc5", b"\x01\x00", True) + _runtest("f", 2**16 - 4, b"\xc5", b"\xff\xfc", True) def test_bin32_from_byte(): - _runtest("B", 2 ** 16, b"\xc6", b"\x00\x01\x00\x00", True) + _runtest("B", 2**16, b"\xc6", b"\x00\x01\x00\x00", True) def test_bin32_from_float(): - _runtest("f", 2 ** 16, b"\xc6", b"\x00\x01\x00\x00", True) + _runtest("f", 2**16, b"\xc6", b"\x00\x01\x00\x00", True) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 6293a45..9f20c07 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -118,8 +118,8 @@ def test_issue124(): def test_unpack_tell(): stream = io.BytesIO() - messages = [2 ** i - 1 for i in range(65)] - messages += [-(2 ** i) for i in range(1, 64)] + messages = [2**i - 1 for i in range(65)] + messages += [-(2**i) for i in range(1, 64)] messages += [ b"hello", b"hello" * 1000, diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 4e26489..253228e 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -10,31 +10,31 @@ if sys.version_info[0] > 2: def test_timestamp(): # timestamp32 - ts = Timestamp(2 ** 32 - 1) + ts = Timestamp(2**32 - 1) assert ts.to_bytes() == b"\xff\xff\xff\xff" packed = msgpack.packb(ts) assert packed == b"\xd6\xff" + ts.to_bytes() unpacked = msgpack.unpackb(packed) assert ts == unpacked - assert ts.seconds == 2 ** 32 - 1 and ts.nanoseconds == 0 + assert ts.seconds == 2**32 - 1 and ts.nanoseconds == 0 # timestamp64 - ts = Timestamp(2 ** 34 - 1, 999999999) + ts = Timestamp(2**34 - 1, 999999999) assert ts.to_bytes() == b"\xee\x6b\x27\xff\xff\xff\xff\xff" packed = msgpack.packb(ts) assert packed == b"\xd7\xff" + ts.to_bytes() unpacked = msgpack.unpackb(packed) assert ts == unpacked - assert ts.seconds == 2 ** 34 - 1 and ts.nanoseconds == 999999999 + assert ts.seconds == 2**34 - 1 and ts.nanoseconds == 999999999 # timestamp96 - ts = Timestamp(2 ** 63 - 1, 999999999) + ts = Timestamp(2**63 - 1, 999999999) assert ts.to_bytes() == b"\x3b\x9a\xc9\xff\x7f\xff\xff\xff\xff\xff\xff\xff" packed = msgpack.packb(ts) assert packed == b"\xc7\x0c\xff" + ts.to_bytes() unpacked = msgpack.unpackb(packed) assert ts == unpacked - assert ts.seconds == 2 ** 63 - 1 and ts.nanoseconds == 999999999 + assert ts.seconds == 2**63 - 1 and ts.nanoseconds == 999999999 # negative fractional ts = Timestamp.from_unix(-2.3) # s: -3, ns: 700000000 From 849c8063817894f7dba166a19fbdbb9ffd8c2b80 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2022 03:23:11 +0100 Subject: [PATCH 1093/1172] Use PyFloat_Pack8() on Python 3.11a7 (#499) Python 3.11a7 adds public functions: * PyFloat_Pack4(), PyFloat_Pack8() * PyFloat_Unpack4(), PyFloat_Unpack8() https://bugs.python.org/issue46906 --- msgpack/pack_template.h | 9 +++++++++ msgpack/unpack_template.h | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 0e940b8..7d479b6 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -568,7 +568,12 @@ static inline int msgpack_pack_float(msgpack_packer* x, float d) { unsigned char buf[5]; buf[0] = 0xca; + +#if PY_VERSION_HEX >= 0x030B00A7 + PyFloat_Pack4(d, (char *)&buf[1], 0); +#else _PyFloat_Pack4(d, &buf[1], 0); +#endif msgpack_pack_append_buffer(x, buf, 5); } @@ -576,7 +581,11 @@ static inline int msgpack_pack_double(msgpack_packer* x, double d) { unsigned char buf[9]; buf[0] = 0xcb; +#if PY_VERSION_HEX >= 0x030B00A7 + PyFloat_Pack8(d, (char *)&buf[1], 0); +#else _PyFloat_Pack8(d, &buf[1], 0); +#endif msgpack_pack_append_buffer(x, buf, 9); } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 9924b9c..8b9fcc1 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -243,10 +243,20 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize _msgpack_load32(uint32_t,n)+1, _ext_zero); case CS_FLOAT: { - double f = _PyFloat_Unpack4((unsigned char*)n, 0); + double f; +#if PY_VERSION_HEX >= 0x030B00A7 + f = PyFloat_Unpack4((const char*)n, 0); +#else + f = _PyFloat_Unpack4((unsigned char*)n, 0); +#endif push_fixed_value(_float, f); } case CS_DOUBLE: { - double f = _PyFloat_Unpack8((unsigned char*)n, 0); + double f; +#if PY_VERSION_HEX >= 0x030B00A7 + f = PyFloat_Unpack8((const char*)n, 0); +#else + f = _PyFloat_Unpack8((unsigned char*)n, 0); +#endif push_fixed_value(_double, f); } case CS_UINT_8: push_fixed_value(_uint8, *(uint8_t*)n); From 6a721faa778e4db23f76bc8a0f9adf85f59f69c7 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 2 May 2022 02:26:53 -0600 Subject: [PATCH 1094/1172] Upgrade black (#505) --- .github/workflows/black.yaml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index be137ae..0a0a737 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -21,5 +21,5 @@ jobs: - name: Black Code Formatter run: | - pip install black==22.1.0 + pip install black==22.3.0 black -S --diff --check msgpack/ test/ setup.py diff --git a/requirements.txt b/requirements.txt index 9bdb478..f557888 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ Cython~=0.29.13 # dev only tools. no need to add pyproject -black==22.1.0 +black==22.3.0 From b901b179d1976e09eb074803dcbb17d8d150c69f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 23 May 2022 04:52:09 +0000 Subject: [PATCH 1095/1172] Update Cython to 0.29.30 --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a9eb8aa..195795f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ # Also declared in requirements.txt, if updating here please also update # there - "Cython~=0.29.13", + "Cython~=0.29.30", "setuptools >= 35.0.2", ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index f557888..9f3c1a0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Also declared in pyproject.toml, if updating here please also update there -Cython~=0.29.13 +Cython~=0.29.30 # dev only tools. no need to add pyproject black==22.3.0 From b75e3412fb8a2b6d6cd1da1b7063e14f6bfc0337 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 23 May 2022 05:01:08 +0000 Subject: [PATCH 1096/1172] Fix pip upgrade --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6e497e0..2a01926 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: - name: Build shell: bash run: | - pip install -U pip + python -m pip install -U pip pip install -r requirements.txt pytest make cython pip install . From 500a238028bdebe123b502b07769578b5f0e8a3a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 24 May 2022 19:46:51 +0900 Subject: [PATCH 1097/1172] Fix Unpacker max_buffer_length handling (#506) --- msgpack/_unpacker.pyx | 28 ++++++++++++---------------- msgpack/fallback.py | 2 ++ test/test_sequnpack.py | 11 ++++++++++- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 27facc0..8b06661 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -440,34 +440,30 @@ cdef class Unpacker(object): self.buf_size = buf_size self.buf_tail = tail + _buf_len - cdef read_from_file(self): - next_bytes = self.file_like_read( - min(self.read_size, - self.max_buffer_size - (self.buf_tail - self.buf_head) - )) + cdef int read_from_file(self) except -1: + cdef Py_ssize_t remains = self.max_buffer_size - (self.buf_tail - self.buf_head) + if remains <= 0: + raise BufferFull + + next_bytes = self.file_like_read(min(self.read_size, remains)) if next_bytes: self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes)) else: self.file_like = None + return 0 cdef object _unpack(self, execute_fn execute, bint iter=0): cdef int ret cdef object obj cdef Py_ssize_t prev_head - if self.buf_head >= self.buf_tail and self.file_like is not None: - self.read_from_file() - while 1: prev_head = self.buf_head - if prev_head >= self.buf_tail: - if iter: - raise StopIteration("No more data to unpack.") - else: - raise OutOfData("No more data to unpack.") - - ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) - self.stream_offset += self.buf_head - prev_head + if prev_head < self.buf_tail: + ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + self.stream_offset += self.buf_head - prev_head + else: + ret = 0 if ret == 1: obj = unpack_data(&self.ctx) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 5f215e9..f560c7b 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -423,6 +423,8 @@ class Unpacker(object): # Read from file remain_bytes = -remain_bytes + if remain_bytes + len(self._buffer) > self._max_buffer_size: + raise BufferFull while remain_bytes > 0: to_read_bytes = max(self._read_size, remain_bytes) read_data = self.file_like.read(to_read_bytes) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 9f20c07..c091076 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -2,7 +2,7 @@ # coding: utf-8 import io from msgpack import Unpacker, BufferFull -from msgpack import pack +from msgpack import pack, packb from msgpack.exceptions import OutOfData from pytest import raises @@ -78,6 +78,15 @@ def test_maxbuffersize(): assert ord("b") == next(unpacker) +def test_maxbuffersize_file(): + buff = io.BytesIO(packb(b"a" * 10) + packb([b"a" * 20] * 2)) + unpacker = Unpacker(buff, read_size=1, max_buffer_size=19, max_bin_len=20) + assert unpacker.unpack() == b"a" * 10 + # assert unpacker.unpack() == [b"a" * 20]*2 + with raises(BufferFull): + print(unpacker.unpack()) + + def test_readbytes(): unpacker = Unpacker(read_size=3) unpacker.feed(b"foobar") From 63837a44d855017f1b2f667afa5ac684fd65591d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 24 May 2022 20:13:07 +0900 Subject: [PATCH 1098/1172] ci: Update action versions. (#507) --- .github/workflows/test.yml | 7 +++---- .github/workflows/wheel.yml | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a01926..2a796aa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,17 +10,17 @@ jobs: strategy: matrix: os: [ubuntu-20.04, windows-2022, macos-10.15] - py: ["3.10", "3.9", "3.8", "3.7", "3.6"] + py: ["3.11-dev", "3.10", "3.9", "3.8", "3.7", "3.6"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.py }} cache: "pip" @@ -28,7 +28,6 @@ jobs: - name: Build shell: bash run: | - python -m pip install -U pip pip install -r requirements.txt pytest make cython pip install . diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 5627b5f..541654d 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up QEMU if: runner.os == 'Linux' @@ -23,7 +23,7 @@ jobs: platforms: arm64 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.9 cache: "pip" @@ -35,7 +35,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.2.2 + uses: pypa/cibuildwheel@v2.5.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" From a34dc945bfe39c1f2f91af2403d906069613ea41 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 25 May 2022 10:00:57 +0900 Subject: [PATCH 1099/1172] 1.0.4rc1 --- msgpack/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 1929df3..81b2e67 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -6,8 +6,8 @@ import os import sys -version = (1, 0, 4, 'dev') -__version__ = "1.0.4dev" +version = (1, 0, 4, 'rc1') +__version__ = "1.0.4rc1" if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: From caadbf2df5a87039a52a5dcf4fc3f151bba70eed Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 25 May 2022 12:10:47 +0900 Subject: [PATCH 1100/1172] Use Actions to publish to PyPI --- .github/workflows/test.yml | 5 +++++ .github/workflows/wheel.yml | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a796aa..d01d74c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,3 +42,8 @@ jobs: run: | MSGPACK_PUREPYTHON=1 pytest -v test + - name: Publish Wheels to TestPyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 541654d..9e8dce8 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -43,9 +43,15 @@ jobs: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_SKIP: pp* - - name: Upload Wheels + - name: Upload Wheels to artifact uses: actions/upload-artifact@v1 with: name: Wheels path: wheelhouse + - name: Publish Wheels to TestPyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages_dir: wheelhouse + password: ${{ secrets.PYPI_API_TOKEN }} From b5acfd53833c3dbd379e539cc6e540cec83d0a99 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 Jun 2022 13:46:51 +0900 Subject: [PATCH 1101/1172] Release v1.0.4 (#509) --- .github/workflows/wheel.yml | 2 +- ChangeLog.rst | 10 ++++++++++ msgpack/__init__.py | 4 ++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 9e8dce8..d73898c 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -35,7 +35,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.5.0 + uses: pypa/cibuildwheel@v2.6.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" diff --git a/ChangeLog.rst b/ChangeLog.rst index fc6df68..a11c814 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,13 @@ +1.0.4 +===== + +Release Date: 2022-06-03 + +* Support Python 3.11 (beta). +* Don't define `__*_ENDIAN__` macro on Unix. by @methane in https://github.com/msgpack/msgpack-python/pull/495 +* Use PyFloat_Pack8() on Python 3.11a7 by @vstinner in https://github.com/msgpack/msgpack-python/pull/499 +* Fix Unpacker max_buffer_length handling by @methane in https://github.com/msgpack/msgpack-python/pull/506 + 1.0.3 ===== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 81b2e67..5071021 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -6,8 +6,8 @@ import os import sys -version = (1, 0, 4, 'rc1') -__version__ = "1.0.4rc1" +version = (1, 0, 4) +__version__ = "1.0.4" if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: From 9d45926a596028e39ec59dd909a56eb5e9e8fee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kul=C3=ADk?= Date: Tue, 2 Aug 2022 06:19:56 +0200 Subject: [PATCH 1102/1172] Usef `__BYTE_ORDER__` instead of `__BYTE_ORDER` (#513) __BYTE_ORDER__ is common predefined macro available on at least gcc and clang. __BYTE_ORDER is macro defined in platform specific headers. --- msgpack/sysdep.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h index ed9c1bc..ae28f0c 100644 --- a/msgpack/sysdep.h +++ b/msgpack/sysdep.h @@ -61,14 +61,12 @@ typedef unsigned int _msgpack_atomic_counter_t; #endif #endif -#else -#include /* __BYTE_ORDER */ #endif #if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define __BIG_ENDIAN__ #elif _WIN32 #define __LITTLE_ENDIAN__ From edca770071fc702e0b4c33f87fb0fa3682b486b4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 8 Aug 2022 15:08:40 +0900 Subject: [PATCH 1103/1172] Fix build error caused by ntohs, ntohl (#514) --- msgpack/sysdep.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h index ae28f0c..7067300 100644 --- a/msgpack/sysdep.h +++ b/msgpack/sysdep.h @@ -61,6 +61,8 @@ typedef unsigned int _msgpack_atomic_counter_t; #endif #endif +#else /* _WIN32 */ +#include /* ntohs, ntohl */ #endif #if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) @@ -93,7 +95,7 @@ typedef unsigned int _msgpack_atomic_counter_t; #ifdef _WIN32 # if defined(ntohl) # define _msgpack_be32(x) ntohl(x) -# elif defined(_byteswap_ulong) || (defined(_MSC_VER) && _MSC_VER >= 1400) +# elif defined(_byteswap_ulong) || defined(_MSC_VER) # define _msgpack_be32(x) ((uint32_t)_byteswap_ulong((unsigned long)x)) # else # define _msgpack_be32(x) \ @@ -106,7 +108,7 @@ typedef unsigned int _msgpack_atomic_counter_t; # define _msgpack_be32(x) ntohl(x) #endif -#if defined(_byteswap_uint64) || (defined(_MSC_VER) && _MSC_VER >= 1400) +#if defined(_byteswap_uint64) || defined(_MSC_VER) # define _msgpack_be64(x) (_byteswap_uint64(x)) #elif defined(bswap_64) # define _msgpack_be64(x) bswap_64(x) From 44a80603838ea480e66c9235036ff742f4013200 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Fri, 9 Sep 2022 09:16:12 +0200 Subject: [PATCH 1104/1172] Add python 3.11 wheels (#517) --- .github/workflows/wheel.yml | 4 ++-- setup.cfg | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index d73898c..770b565 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -8,7 +8,7 @@ jobs: build_wheels: strategy: matrix: - os: [ubuntu-20.04, windows-2022, macos-10.15] + os: [ubuntu-22.04, windows-2022, macos-10.15] runs-on: ${{ matrix.os }} name: Build wheels on ${{ matrix.os }} @@ -35,7 +35,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.6.0 + uses: pypa/cibuildwheel@v2.9.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" diff --git a/setup.cfg b/setup.cfg index 1cb6ce3..e378284 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,6 +22,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy Intended Audience :: Developers From c3995669f1f821596714240c2cd07943810f8658 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 18 Jan 2023 08:08:58 +0000 Subject: [PATCH 1105/1172] Remove unused code --- msgpack/pack.h | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/msgpack/pack.h b/msgpack/pack.h index 4f3ce1d..1e849ac 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -71,7 +71,6 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ static inline int msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit) { -#if PY_MAJOR_VERSION >= 3 assert(PyUnicode_Check(o)); Py_ssize_t len; @@ -87,31 +86,6 @@ msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit) if (ret) return ret; return msgpack_pack_raw_body(pk, buf, len); -#else - PyObject *bytes; - Py_ssize_t len; - int ret; - - // py2 - bytes = PyUnicode_AsUTF8String(o); - if (bytes == NULL) - return -1; - - len = PyString_GET_SIZE(bytes); - if (len > limit) { - Py_DECREF(bytes); - return -2; - } - - ret = msgpack_pack_raw(pk, len); - if (ret) { - Py_DECREF(bytes); - return -1; - } - ret = msgpack_pack_raw_body(pk, PyString_AS_STRING(bytes), len); - Py_DECREF(bytes); - return ret; -#endif } #ifdef __cplusplus From b82d0b62f187552b8108602d7b0451ac362a29cc Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 18 Jan 2023 19:13:44 +0900 Subject: [PATCH 1106/1172] fallback: Fix packing multidim memoryview (#527) Fix #526 --- msgpack/fallback.py | 2 +- test/test_memoryview.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index f560c7b..e8cebc1 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -814,7 +814,7 @@ class Packer(object): self._pack_raw_header(n) return self._buffer.write(obj) if check(obj, memoryview): - n = len(obj) * obj.itemsize + n = obj.nbytes if n >= 2**32: raise ValueError("Memoryview is too large") self._pack_bin_header(n) diff --git a/test/test_memoryview.py b/test/test_memoryview.py index 84941db..a0939a6 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -96,3 +96,11 @@ def test_bin32_from_byte(): def test_bin32_from_float(): _runtest("f", 2**16, b"\xc6", b"\x00\x01\x00\x00", True) + + +def test_multidim_memoryview(): + # See https://github.com/msgpack/msgpack-python/issues/526 + view = memoryview(b"\00" * 6) + data = view.cast(view.format, (3, 2)) + packed = packb(data) + assert packed == b'\xc4\x06\x00\x00\x00\x00\x00\x00' From 10082295536098d90681da5d7199ca384e8b8ff8 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 18 Jan 2023 19:47:15 +0900 Subject: [PATCH 1107/1172] Release v1.0.5rc1 (#528) --- .github/workflows/black.yaml | 4 ++-- .github/workflows/test.yml | 6 +++--- .github/workflows/wheel.yml | 2 +- ChangeLog.rst | 9 +++++++++ msgpack/__init__.py | 4 ++-- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index 0a0a737..1e28b7b 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -11,13 +11,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.x' architecture: 'x64' - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Black Code Formatter run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d01d74c..5e41167 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,8 +9,8 @@ jobs: test: strategy: matrix: - os: [ubuntu-20.04, windows-2022, macos-10.15] - py: ["3.11-dev", "3.10", "3.9", "3.8", "3.7", "3.6"] + os: [ubuntu-22.04, windows-2022, macos-10.15] + py: ["3.11", "3.10", "3.9", "3.8", "3.7"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} cache: "pip" diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 770b565..b2879fe 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -35,7 +35,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.9.0 + uses: pypa/cibuildwheel@v2.12.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" diff --git a/ChangeLog.rst b/ChangeLog.rst index a11c814..1b7ae2c 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,12 @@ +1.0.5rc1 +======== + +Release Date: 2023-01-18 + +* Use ``__BYTE_ORDER__`` instead of ``__BYTE_ORDER`` for portability. (#513, #514) +* Add Python 3.11 wheels (#517) +* fallback: Fix packing multidimensional memoryview (#527) + 1.0.4 ===== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 5071021..501b291 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -6,8 +6,8 @@ import os import sys -version = (1, 0, 4) -__version__ = "1.0.4" +version = (1, 0, 5, 'rc', 1) +__version__ = "1.0.5rc1" if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: From e3ef909c47e5a245fc9d711e9d974a5f9df99303 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 18 Jan 2023 13:07:24 +0000 Subject: [PATCH 1108/1172] Action: Use setup-python@v4 --- .github/workflows/wheel.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index b2879fe..2305008 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -22,10 +22,10 @@ jobs: with: platforms: arm64 - - name: Set up Python 3.9 - uses: actions/setup-python@v3 + - name: Set up Python 3.x + uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: "3.x" cache: "pip" - name: Prepare From dcb775031c0b1d575b90e822e81e845ebfda4a2e Mon Sep 17 00:00:00 2001 From: Anthon van der Neut Date: Sun, 5 Mar 2023 15:45:38 +0100 Subject: [PATCH 1109/1172] minor type in exception message (#533) interger -> integer --- msgpack/ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgpack/ext.py b/msgpack/ext.py index 25544c5..23e0d6b 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -56,7 +56,7 @@ class Timestamp(object): Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns. """ if not isinstance(seconds, int_types): - raise TypeError("seconds must be an interger") + raise TypeError("seconds must be an integer") if not isinstance(nanoseconds, int_types): raise TypeError("nanoseconds must be an integer") if not (0 <= nanoseconds < 10**9): From aa9ce3e2bbc1d3d0476396892c46e704292455ab Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 Jun 2022 14:37:21 +0900 Subject: [PATCH 1110/1172] Action: Run publish on tag creation. --- .github/workflows/wheel.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 2305008..5f103a3 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -49,8 +49,8 @@ jobs: name: Wheels path: wheelhouse - - name: Publish Wheels to TestPyPI - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + - name: Publish Wheels to PyPI + if: github.event_name == 'create' && github.event.ref_type == 'tag' uses: pypa/gh-action-pypi-publish@release/v1 with: packages_dir: wheelhouse From 4c55f809fe2231130cf99b20538b26b92f1bea31 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Mar 2023 00:43:28 +0900 Subject: [PATCH 1111/1172] Release v1.0.5 (#534) --- ChangeLog.rst | 6 +++--- msgpack/__init__.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 1b7ae2c..5354799 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,7 +1,7 @@ -1.0.5rc1 -======== +1.0.5 +===== -Release Date: 2023-01-18 +Release Date: 2023-03-08 * Use ``__BYTE_ORDER__`` instead of ``__BYTE_ORDER`` for portability. (#513, #514) * Add Python 3.11 wheels (#517) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 501b291..1300b86 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -6,8 +6,8 @@ import os import sys -version = (1, 0, 5, 'rc', 1) -__version__ = "1.0.5rc1" +version = (1, 0, 5) +__version__ = "1.0.5" if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: From 35b2d246cfdb19484caa5789512bf71ee378caec Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Mar 2023 00:47:52 +0900 Subject: [PATCH 1112/1172] Action: Update wheel workflow --- .github/workflows/wheel.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 5f103a3..6cf2fe9 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -8,7 +8,7 @@ jobs: build_wheels: strategy: matrix: - os: [ubuntu-22.04, windows-2022, macos-10.15] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] runs-on: ${{ matrix.os }} name: Build wheels on ${{ matrix.os }} @@ -48,10 +48,3 @@ jobs: with: name: Wheels path: wheelhouse - - - name: Publish Wheels to PyPI - if: github.event_name == 'create' && github.event.ref_type == 'tag' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages_dir: wheelhouse - password: ${{ secrets.PYPI_API_TOKEN }} From 0516c2c2a97ef48a2becf30bc8b2365ca16199f1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Mar 2023 01:22:38 +0900 Subject: [PATCH 1113/1172] Action: Update test workflow --- .github/workflows/test.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5e41167..8878102 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ jobs: test: strategy: matrix: - os: [ubuntu-22.04, windows-2022, macos-10.15] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] py: ["3.11", "3.10", "3.9", "3.8", "3.7"] runs-on: ${{ matrix.os }} @@ -41,9 +41,3 @@ jobs: shell: bash run: | MSGPACK_PUREPYTHON=1 pytest -v test - - - name: Publish Wheels to TestPyPI - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} From 802cbc9495ed059b62b46f057cf3d71f756e2480 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 1 Apr 2023 00:02:25 +0900 Subject: [PATCH 1114/1172] Add security policy --- SECURITY.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..75f0c54 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,5 @@ +## Security contact information + +To report a security vulnerability, please use the +[Tidelift security contact](https://tidelift.com/security). +Tidelift will coordinate the fix and disclosure. \ No newline at end of file From 45f848695c855966d8a46656d8de1a2734d934ae Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Sat, 8 Apr 2023 02:18:25 -0300 Subject: [PATCH 1115/1172] fix: build status badge (#538) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cb81648..7f7c423 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MessagePack for Python -[![Build Status](https://travis-ci.org/msgpack/msgpack-python.svg?branch=master)](https://travis-ci.org/msgpack/msgpack-python) +[![Build Status](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml/badge.svg)](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml) [![Documentation Status](https://readthedocs.org/projects/msgpack-python/badge/?version=latest)](https://msgpack-python.readthedocs.io/en/latest/?badge=latest) ## What's this From feec06206c7f8dd8efeac6177badbf5b256b36e4 Mon Sep 17 00:00:00 2001 From: sblondon Date: Sun, 21 May 2023 09:26:39 +0200 Subject: [PATCH 1116/1172] Drop python2 support (#519) The PR removes python2 references and cases. Close #518 Co-authored-by: Inada Naoki --- README.md | 4 +-- docs/conf.py | 20 ++++++------ msgpack/__init__.py | 2 +- msgpack/_packer.pyx | 1 - msgpack/_unpacker.pyx | 2 +- msgpack/ext.py | 24 +++----------- msgpack/fallback.py | 70 ++++++++++++----------------------------- setup.py | 3 +- test/test_buffer.py | 1 - test/test_case.py | 2 +- test/test_except.py | 2 +- test/test_extension.py | 15 ++------- test/test_memoryview.py | 5 --- test/test_pack.py | 6 ---- test/test_timestamp.py | 27 +++++++--------- test/test_unpack.py | 4 +-- tox.ini | 13 -------- 17 files changed, 58 insertions(+), 143 deletions(-) diff --git a/README.md b/README.md index 7f7c423..61f99e1 100644 --- a/README.md +++ b/README.md @@ -220,9 +220,9 @@ and `raw=True` options. ```pycon >>> import msgpack ->>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=False), raw=True) +>>> msgpack.unpackb(msgpack.packb([b'spam', 'eggs'], use_bin_type=False), raw=True) [b'spam', b'eggs'] ->>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), raw=False) +>>> msgpack.unpackb(msgpack.packb([b'spam', 'eggs'], use_bin_type=True), raw=False) [b'spam', 'eggs'] ``` diff --git a/docs/conf.py b/docs/conf.py index 6b432be..91ce77f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,8 +40,8 @@ source_suffix = ".rst" master_doc = "index" # General information about the project. -project = u"msgpack" -copyright = u"Inada Naoki" +project = "msgpack" +copyright = "Inada Naoki" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -181,7 +181,7 @@ latex_elements = { # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ("index", "msgpack.tex", u"msgpack Documentation", u"Author", "manual"), + ("index", "msgpack.tex", "msgpack Documentation", "Author", "manual"), ] # The name of an image file (relative to this directory) to place at the top of @@ -209,7 +209,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [("index", "msgpack", u"msgpack Documentation", [u"Author"], 1)] +man_pages = [("index", "msgpack", "msgpack Documentation", ["Author"], 1)] # If true, show URL addresses after external links. # man_show_urls = False @@ -224,8 +224,8 @@ texinfo_documents = [ ( "index", "msgpack", - u"msgpack Documentation", - u"Author", + "msgpack Documentation", + "Author", "msgpack", "One line description of project.", "Miscellaneous", @@ -245,10 +245,10 @@ texinfo_documents = [ # -- Options for Epub output --------------------------------------------------- # Bibliographic Dublin Core info. -epub_title = u"msgpack" -epub_author = u"Author" -epub_publisher = u"Author" -epub_copyright = u"2013, Author" +epub_title = "msgpack" +epub_author = "Author" +epub_publisher = "Author" +epub_copyright = "2013, Author" # The language of the text. It defaults to the language option # or en if the language is not set. diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 1300b86..6c10dc2 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -10,7 +10,7 @@ version = (1, 0, 5) __version__ = "1.0.5" -if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2: +if os.environ.get("MSGPACK_PUREPYTHON"): from .fallback import Packer, unpackb, Unpacker else: try: diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 396da0c..074b39f 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -98,7 +98,6 @@ cdef class Packer(object): If set to true, datetime with tzinfo is packed into Timestamp type. Note that the tzinfo is stripped in the timestamp. You can get UTC datetime with `timestamp=3` option of the Unpacker. - (Python 2 is not supported). :param str unicode_errors: The error handler for encoding unicode. (default: 'strict') diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 8b06661..d5dc5ea 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -236,7 +236,7 @@ cdef class Unpacker(object): 0 - Timestamp 1 - float (Seconds from the EPOCH) 2 - int (Nanoseconds from the EPOCH) - 3 - datetime.datetime (UTC). Python 2 is not supported. + 3 - datetime.datetime (UTC). :param bool strict_map_key: If true (default), only str or bytes are accepted for map (dict) keys. diff --git a/msgpack/ext.py b/msgpack/ext.py index 23e0d6b..07f96a5 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -5,19 +5,6 @@ import sys import struct -PY2 = sys.version_info[0] == 2 - -if PY2: - int_types = (int, long) - _utc = None -else: - int_types = int - try: - _utc = datetime.timezone.utc - except AttributeError: - _utc = datetime.timezone(datetime.timedelta(0)) - - class ExtType(namedtuple("ExtType", "code data")): """ExtType represents ext type in msgpack.""" @@ -55,9 +42,9 @@ class Timestamp(object): Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns. """ - if not isinstance(seconds, int_types): + if not isinstance(seconds, int): raise TypeError("seconds must be an integer") - if not isinstance(nanoseconds, int_types): + if not isinstance(nanoseconds, int): raise TypeError("nanoseconds must be an integer") if not (0 <= nanoseconds < 10**9): raise ValueError( @@ -174,11 +161,10 @@ class Timestamp(object): def to_datetime(self): """Get the timestamp as a UTC datetime. - Python 2 is not supported. - :rtype: datetime. """ - return datetime.datetime.fromtimestamp(0, _utc) + datetime.timedelta( + utc = datetime.timezone.utc + return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta( seconds=self.to_unix() ) @@ -186,8 +172,6 @@ class Timestamp(object): def from_datetime(dt): """Create a Timestamp from datetime with tzinfo. - Python 2 is not supported. - :rtype: Timestamp """ return Timestamp.from_unix(dt.timestamp()) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index e8cebc1..618c362 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -4,22 +4,6 @@ import sys import struct -PY2 = sys.version_info[0] == 2 -if PY2: - int_types = (int, long) - - def dict_iteritems(d): - return d.iteritems() - -else: - int_types = int - unicode = str - xrange = range - - def dict_iteritems(d): - return d.items() - - if sys.version_info < (3, 5): # Ugly hack... RecursionError = RuntimeError @@ -134,15 +118,6 @@ def unpackb(packed, **kwargs): return ret -if sys.version_info < (2, 7, 6): - - def _unpack_from(f, b, o=0): - """Explicit type cast for legacy struct.unpack_from""" - return struct.unpack_from(f, bytes(b), o) - -else: - _unpack_from = struct.unpack_from - _NO_FORMAT_USED = "" _MSGPACK_HEADERS = { 0xC4: (1, _NO_FORMAT_USED, TYPE_BIN), @@ -202,7 +177,7 @@ class Unpacker(object): 0 - Timestamp 1 - float (Seconds from the EPOCH) 2 - int (Nanoseconds from the EPOCH) - 3 - datetime.datetime (UTC). Python 2 is not supported. + 3 - datetime.datetime (UTC). :param bool strict_map_key: If true (default), only str or bytes are accepted for map (dict) keys. @@ -477,7 +452,7 @@ class Unpacker(object): size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) if len(fmt) > 0: - n = _unpack_from(fmt, self._buffer, self._buff_i)[0] + n = struct.unpack_from(fmt, self._buffer, self._buff_i)[0] else: n = self._buffer[self._buff_i] self._buff_i += size @@ -487,7 +462,7 @@ class Unpacker(object): elif 0xC7 <= b <= 0xC9: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) - L, n = _unpack_from(fmt, self._buffer, self._buff_i) + L, n = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if L > self._max_ext_len: raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) @@ -496,7 +471,7 @@ class Unpacker(object): size, fmt = _MSGPACK_HEADERS[b] self._reserve(size) if len(fmt) > 0: - obj = _unpack_from(fmt, self._buffer, self._buff_i)[0] + obj = struct.unpack_from(fmt, self._buffer, self._buff_i)[0] else: obj = self._buffer[self._buff_i] self._buff_i += size @@ -507,13 +482,13 @@ class Unpacker(object): "%s exceeds max_ext_len(%s)" % (size, self._max_ext_len) ) self._reserve(size + 1) - n, obj = _unpack_from(fmt, self._buffer, self._buff_i) + n, obj = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size + 1 elif 0xD9 <= b <= 0xDB: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) if len(fmt) > 0: - (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i) else: n = self._buffer[self._buff_i] self._buff_i += size @@ -523,7 +498,7 @@ class Unpacker(object): elif 0xDC <= b <= 0xDD: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) - (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_array_len: raise ValueError( @@ -532,7 +507,7 @@ class Unpacker(object): elif 0xDE <= b <= 0xDF: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) - (n,) = _unpack_from(fmt, self._buffer, self._buff_i) + (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_map_len: raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) @@ -554,12 +529,12 @@ class Unpacker(object): # TODO should we eliminate the recursion? if typ == TYPE_ARRAY: if execute == EX_SKIP: - for i in xrange(n): + for i in range(n): # TODO check whether we need to call `list_hook` self._unpack(EX_SKIP) return ret = newlist_hint(n) - for i in xrange(n): + for i in range(n): ret.append(self._unpack(EX_CONSTRUCT)) if self._list_hook is not None: ret = self._list_hook(ret) @@ -567,7 +542,7 @@ class Unpacker(object): return ret if self._use_list else tuple(ret) if typ == TYPE_MAP: if execute == EX_SKIP: - for i in xrange(n): + for i in range(n): # TODO check whether we need to call hooks self._unpack(EX_SKIP) self._unpack(EX_SKIP) @@ -575,17 +550,17 @@ class Unpacker(object): if self._object_pairs_hook is not None: ret = self._object_pairs_hook( (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT)) - for _ in xrange(n) + for _ in range(n) ) else: ret = {} - for _ in xrange(n): + for _ in range(n): key = self._unpack(EX_CONSTRUCT) - if self._strict_map_key and type(key) not in (unicode, bytes): + if self._strict_map_key and type(key) not in (str, bytes): raise ValueError( "%s is not allowed for map key" % str(type(key)) ) - if not PY2 and type(key) is str: + if type(key) is str: key = sys.intern(key) ret[key] = self._unpack(EX_CONSTRUCT) if self._object_hook is not None: @@ -698,7 +673,6 @@ class Packer(object): If set to true, datetime with tzinfo is packed into Timestamp type. Note that the tzinfo is stripped in the timestamp. You can get UTC datetime with `timestamp=3` option of the Unpacker. - (Python 2 is not supported). :param str unicode_errors: The error handler for encoding unicode. (default: 'strict') @@ -743,8 +717,6 @@ class Packer(object): self._autoreset = autoreset self._use_bin_type = use_bin_type self._buffer = StringIO() - if PY2 and datetime: - raise ValueError("datetime is not supported in Python 2") self._datetime = bool(datetime) self._unicode_errors = unicode_errors or "strict" if default is not None: @@ -774,7 +746,7 @@ class Packer(object): if obj: return self._buffer.write(b"\xc3") return self._buffer.write(b"\xc2") - if check(obj, int_types): + if check(obj, int): if 0 <= obj < 0x80: return self._buffer.write(struct.pack("B", obj)) if -0x20 <= obj < 0: @@ -806,7 +778,7 @@ class Packer(object): raise ValueError("%s is too large" % type(obj).__name__) self._pack_bin_header(n) return self._buffer.write(obj) - if check(obj, unicode): + if check(obj, str): obj = obj.encode("utf-8", self._unicode_errors) n = len(obj) if n >= 2**32: @@ -855,13 +827,11 @@ class Packer(object): if check(obj, list_types): n = len(obj) self._pack_array_header(n) - for i in xrange(n): + for i in range(n): self._pack(obj[i], nest_limit - 1) return if check(obj, dict): - return self._pack_map_pairs( - len(obj), dict_iteritems(obj), nest_limit - 1 - ) + return self._pack_map_pairs(len(obj), obj.items(), nest_limit - 1) if self._datetime and check(obj, _DateTime) and obj.tzinfo is not None: obj = Timestamp.from_datetime(obj) @@ -1004,7 +974,7 @@ class Packer(object): def getbuffer(self): """Return view of internal buffer.""" - if USING_STRINGBUILDER or PY2: + if USING_STRINGBUILDER: return memoryview(self.bytes()) else: return self._buffer.getbuffer() diff --git a/setup.py b/setup.py index 9630cda..15ba774 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,6 @@ from setuptools.command.sdist import sdist PYPY = hasattr(sys, "pypy_version_info") -PY2 = sys.version_info[0] == 2 class NoCython(Exception): @@ -79,7 +78,7 @@ if sys.platform == "win32": macros = [("__LITTLE_ENDIAN__", "1")] ext_modules = [] -if not PYPY and not PY2 and not os.environ.get("MSGPACK_PUREPYTHON"): +if not PYPY and not os.environ.get("MSGPACK_PUREPYTHON"): ext_modules.append( Extension( "msgpack._cmsgpack", diff --git a/test/test_buffer.py b/test/test_buffer.py index 62507cf..0447058 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -6,7 +6,6 @@ import pytest from msgpack import packb, unpackb -@pytest.mark.skipif(sys.version_info[0] == 2, reason="Python 2 is not supported") def test_unpack_buffer(): from array import array diff --git a/test/test_case.py b/test/test_case.py index a0a3c5a..1c4e322 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -134,4 +134,4 @@ def test_match(): def test_unicode(): - assert unpackb(packb(u"foobar"), use_list=1) == u"foobar" + assert unpackb(packb("foobar"), use_list=1) == "foobar" diff --git a/test/test_except.py b/test/test_except.py index 5544f2b..745ebec 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -53,7 +53,7 @@ def test_invalidvalue(): def test_strict_map_key(): - valid = {u"unicode": 1, b"bytes": 2} + valid = {"unicode": 1, b"bytes": 2} packed = packb(valid, use_bin_type=True) assert valid == unpackb(packed, raw=False, strict_map_key=True) diff --git a/test/test_extension.py b/test/test_extension.py index 6b36575..dfbe435 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -55,10 +55,7 @@ def test_extension_type(): print("ext_hook called", code, data) assert code == 123 obj = array.array("d") - try: - obj.frombytes(data) - except AttributeError: # PY2 - obj.fromstring(data) + obj.frombytes(data) return obj obj = [42, b"hello", array.array("d", [1.1, 2.2, 3.3])] @@ -67,20 +64,14 @@ def test_extension_type(): assert obj == obj2 -import sys - -if sys.version > "3": - long = int - - def test_overriding_hooks(): def default(obj): - if isinstance(obj, long): + if isinstance(obj, int): return {"__type__": "long", "__data__": str(obj)} else: return obj - obj = {"testval": long(1823746192837461928374619)} + obj = {"testval": 1823746192837461928374619} refobj = {"testval": default(obj["testval"])} refout = msgpack.packb(refobj) assert isinstance(refout, (str, bytes)) diff --git a/test/test_memoryview.py b/test/test_memoryview.py index a0939a6..63beab1 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -7,11 +7,6 @@ from msgpack import packb, unpackb import sys -pytestmark = pytest.mark.skipif( - sys.version_info[0] < 3, reason="Only Python 3 supports buffer protocol" -) - - def make_array(f, data): a = array(f) a.frombytes(data) diff --git a/test/test_pack.py b/test/test_pack.py index a51d84c..65c9cb1 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -80,9 +80,6 @@ def testPackByteArrays(): check(td) -@pytest.mark.skipif( - sys.version_info < (3, 0), reason="Python 2 passes invalid surrogates" -) def testIgnoreUnicodeErrors(): re = unpackb( packb(b"abc\xeddef", use_bin_type=False), raw=False, unicode_errors="ignore" @@ -96,9 +93,6 @@ def testStrictUnicodeUnpack(): unpackb(packed, raw=False, use_list=1) -@pytest.mark.skipif( - sys.version_info < (3, 0), reason="Python 2 passes invalid surrogates" -) def testIgnoreErrorsPack(): re = unpackb( packb("abc\uDC80\uDCFFdef", use_bin_type=True, unicode_errors="ignore"), diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 253228e..af84a2f 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -4,9 +4,6 @@ import datetime import msgpack from msgpack.ext import Timestamp -if sys.version_info[0] > 2: - from msgpack.ext import _utc - def test_timestamp(): # timestamp32 @@ -85,33 +82,33 @@ def test_timestamp_to(): assert t.to_unix_nano() == 42000014000 -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_timestamp_datetime(): t = Timestamp(42, 14) - assert t.to_datetime() == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc) + utc = datetime.timezone.utc + assert t.to_datetime() == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=utc) -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_unpack_datetime(): t = Timestamp(42, 14) + utc = datetime.timezone.utc packed = msgpack.packb(t) unpacked = msgpack.unpackb(packed, timestamp=3) - assert unpacked == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc) + assert unpacked == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=utc) -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_pack_unpack_before_epoch(): - t_in = datetime.datetime(1960, 1, 1, tzinfo=_utc) + utc = datetime.timezone.utc + t_in = datetime.datetime(1960, 1, 1, tzinfo=utc) packed = msgpack.packb(t_in, datetime=True) unpacked = msgpack.unpackb(packed, timestamp=3) assert unpacked == t_in -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_pack_datetime(): t = Timestamp(42, 14000) dt = t.to_datetime() - assert dt == datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=_utc) + utc = datetime.timezone.utc + assert dt == datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=utc) packed = msgpack.packb(dt, datetime=True) packed2 = msgpack.packb(t) @@ -131,10 +128,10 @@ def test_pack_datetime(): assert msgpack.unpackb(packed) is None -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_issue451(): # https://github.com/msgpack/msgpack-python/issues/451 - dt = datetime.datetime(2100, 1, 1, 1, 1, tzinfo=_utc) + utc = datetime.timezone.utc + dt = datetime.datetime(2100, 1, 1, 1, 1, tzinfo=utc) packed = msgpack.packb(dt, datetime=True) assert packed == b"\xd6\xff\xf4\x86eL" @@ -142,7 +139,6 @@ def test_issue451(): assert dt == unpacked -@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only") def test_pack_datetime_without_tzinfo(): dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14) with pytest.raises(ValueError, match="where tzinfo=None"): @@ -152,7 +148,8 @@ def test_pack_datetime_without_tzinfo(): packed = msgpack.packb(dt, datetime=True, default=lambda x: None) assert packed == msgpack.packb(None) - dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=_utc) + utc = datetime.timezone.utc + dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=utc) packed = msgpack.packb(dt, datetime=True) unpacked = msgpack.unpackb(packed, timestamp=3) assert unpacked == dt diff --git a/test/test_unpack.py b/test/test_unpack.py index aa4c01f..c714102 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -70,7 +70,7 @@ def test_unpacker_ext_hook(): def test_unpacker_tell(): - objects = 1, 2, u"abc", u"def", u"ghi" + objects = 1, 2, "abc", "def", "ghi" packed = b"\x01\x02\xa3abc\xa3def\xa3ghi" positions = 1, 2, 6, 10, 14 unpacker = Unpacker(BytesIO(packed)) @@ -80,7 +80,7 @@ def test_unpacker_tell(): def test_unpacker_tell_read_bytes(): - objects = 1, u"abc", u"ghi" + objects = 1, "abc", "ghi" packed = b"\x01\x02\xa3abc\xa3def\xa3ghi" raw_data = b"\x02", b"\xa3def", b"" lenghts = 1, 4, 999 diff --git a/tox.ini b/tox.ini index 29c256d..1ef2d18 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,7 @@ [tox] envlist = - py27-pure, {py35,py36,py37,py38}-{c,pure}, {pypy,pypy3}-pure, - py27-x86, py34-x86, isolated_build = true @@ -19,17 +17,6 @@ commands= setenv= pure: MSGPACK_PUREPYTHON=x -[testenv:py27-x86] -basepython=python2.7-x86 -deps= - pytest - -changedir=test -commands= - python -c 'import sys; print(hex(sys.maxsize))' - python -c 'from msgpack import _cmsgpack' - py.test - [testenv:py34-x86] basepython=python3.4-x86 deps= From c8d0751fe3375a5e8005b5edf955cd5904aaec2f Mon Sep 17 00:00:00 2001 From: Evgeny Markov Date: Tue, 23 May 2023 18:41:08 +0200 Subject: [PATCH 1117/1172] Drop Python 3.6 support (#543) The following steps have been taken: 1. Black was updated to latest version. The code has been formatted with the new version. 2. The pyupgrade utility is installed. This helped to remove all the code that was needed to support Python < 3.7. Fix #541. Co-authored-by: Inada Naoki --- Makefile | 8 ++++- msgpack/__init__.py | 1 - msgpack/ext.py | 21 ++++--------- msgpack/fallback.py | 70 ++++++++++++----------------------------- pyproject.toml | 5 +++ requirements.txt | 8 +++-- setup.cfg | 1 - setup.py | 9 ++---- test/test_buffer.py | 1 - test/test_case.py | 7 ++--- test/test_except.py | 1 - test/test_extension.py | 7 ++--- test/test_format.py | 5 +-- test/test_limits.py | 2 -- test/test_memoryview.py | 1 - test/test_newspec.py | 2 -- test/test_obj.py | 1 - test/test_pack.py | 12 +++---- test/test_seq.py | 1 - test/test_sequnpack.py | 1 - test/test_stricttype.py | 12 +++---- test/test_subtype.py | 1 - test/test_unpack.py | 2 +- 23 files changed, 60 insertions(+), 119 deletions(-) diff --git a/Makefile b/Makefile index 415dcfd..e4f22da 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,16 @@ +PYTHON_SOURCES = msgpack test setup.py + .PHONY: all all: cython python setup.py build_ext -i -f .PHONY: black black: - black -S msgpack/ test/ setup.py + black $(PYTHON_SOURCES) + +.PHONY: pyupgrade +pyupgrade: + @find $(PYTHON_SOURCES) -name '*.py' -type f -exec pyupgrade --py37-plus '{}' \; .PHONY: cython cython: diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 6c10dc2..638236f 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,4 +1,3 @@ -# coding: utf-8 from .exceptions import * from .ext import ExtType, Timestamp diff --git a/msgpack/ext.py b/msgpack/ext.py index 07f96a5..9794294 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -1,4 +1,3 @@ -# coding: utf-8 from collections import namedtuple import datetime import sys @@ -15,10 +14,10 @@ class ExtType(namedtuple("ExtType", "code data")): raise TypeError("data must be bytes") if not 0 <= code <= 127: raise ValueError("code must be 0~127") - return super(ExtType, cls).__new__(cls, code, data) + return super().__new__(cls, code, data) -class Timestamp(object): +class Timestamp: """Timestamp represents the Timestamp extension type in msgpack. When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. When using pure-Python @@ -47,24 +46,18 @@ class Timestamp(object): if not isinstance(nanoseconds, int): raise TypeError("nanoseconds must be an integer") if not (0 <= nanoseconds < 10**9): - raise ValueError( - "nanoseconds must be a non-negative integer less than 999999999." - ) + raise ValueError("nanoseconds must be a non-negative integer less than 999999999.") self.seconds = seconds self.nanoseconds = nanoseconds def __repr__(self): """String representation of Timestamp.""" - return "Timestamp(seconds={0}, nanoseconds={1})".format( - self.seconds, self.nanoseconds - ) + return f"Timestamp(seconds={self.seconds}, nanoseconds={self.nanoseconds})" def __eq__(self, other): """Check for equality with another Timestamp object""" if type(other) is self.__class__: - return ( - self.seconds == other.seconds and self.nanoseconds == other.nanoseconds - ) + return self.seconds == other.seconds and self.nanoseconds == other.nanoseconds return False def __ne__(self, other): @@ -164,9 +157,7 @@ class Timestamp(object): :rtype: datetime. """ utc = datetime.timezone.utc - return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta( - seconds=self.to_unix() - ) + return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta(seconds=self.to_unix()) @staticmethod def from_datetime(dt): diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 618c362..ac1eaf4 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -4,23 +4,6 @@ import sys import struct -if sys.version_info < (3, 5): - # Ugly hack... - RecursionError = RuntimeError - - def _is_recursionerror(e): - return ( - len(e.args) == 1 - and isinstance(e.args[0], str) - and e.args[0].startswith("maximum recursion depth exceeded") - ) - -else: - - def _is_recursionerror(e): - return True - - if hasattr(sys, "pypy_version_info"): # StringIO is slow on PyPy, StringIO is faster. However: PyPy's own # StringBuilder is fastest. @@ -32,7 +15,7 @@ if hasattr(sys, "pypy_version_info"): from __pypy__.builders import StringBuilder USING_STRINGBUILDER = True - class StringIO(object): + class StringIO: def __init__(self, s=b""): if s: self.builder = StringBuilder(len(s)) @@ -109,10 +92,8 @@ def unpackb(packed, **kwargs): ret = unpacker._unpack() except OutOfData: raise ValueError("Unpack failed: incomplete input") - except RecursionError as e: - if _is_recursionerror(e): - raise StackError - raise + except RecursionError: + raise StackError if unpacker._got_extradata(): raise ExtraData(ret, unpacker._get_extradata()) return ret @@ -151,7 +132,7 @@ _MSGPACK_HEADERS = { } -class Unpacker(object): +class Unpacker: """Streaming unpacker. Arguments: @@ -334,9 +315,7 @@ class Unpacker(object): if object_pairs_hook is not None and not callable(object_pairs_hook): raise TypeError("`object_pairs_hook` is not callable") if object_hook is not None and object_pairs_hook is not None: - raise TypeError( - "object_pairs_hook and object_hook are mutually " "exclusive" - ) + raise TypeError("object_pairs_hook and object_hook are mutually exclusive") if not callable(ext_hook): raise TypeError("`ext_hook` is not callable") @@ -428,20 +407,18 @@ class Unpacker(object): n = b & 0b00011111 typ = TYPE_RAW if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len)) + raise ValueError(f"{n} exceeds max_str_len({self._max_str_len})") obj = self._read(n) elif b & 0b11110000 == 0b10010000: n = b & 0b00001111 typ = TYPE_ARRAY if n > self._max_array_len: - raise ValueError( - "%s exceeds max_array_len(%s)" % (n, self._max_array_len) - ) + raise ValueError(f"{n} exceeds max_array_len({self._max_array_len})") elif b & 0b11110000 == 0b10000000: n = b & 0b00001111 typ = TYPE_MAP if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) + raise ValueError(f"{n} exceeds max_map_len({self._max_map_len})") elif b == 0xC0: obj = None elif b == 0xC2: @@ -457,7 +434,7 @@ class Unpacker(object): n = self._buffer[self._buff_i] self._buff_i += size if n > self._max_bin_len: - raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) + raise ValueError(f"{n} exceeds max_bin_len({self._max_bin_len})") obj = self._read(n) elif 0xC7 <= b <= 0xC9: size, fmt, typ = _MSGPACK_HEADERS[b] @@ -465,7 +442,7 @@ class Unpacker(object): L, n = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if L > self._max_ext_len: - raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) + raise ValueError(f"{L} exceeds max_ext_len({self._max_ext_len})") obj = self._read(L) elif 0xCA <= b <= 0xD3: size, fmt = _MSGPACK_HEADERS[b] @@ -478,9 +455,7 @@ class Unpacker(object): elif 0xD4 <= b <= 0xD8: size, fmt, typ = _MSGPACK_HEADERS[b] if self._max_ext_len < size: - raise ValueError( - "%s exceeds max_ext_len(%s)" % (size, self._max_ext_len) - ) + raise ValueError(f"{size} exceeds max_ext_len({self._max_ext_len})") self._reserve(size + 1) n, obj = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size + 1 @@ -493,7 +468,7 @@ class Unpacker(object): n = self._buffer[self._buff_i] self._buff_i += size if n > self._max_str_len: - raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len)) + raise ValueError(f"{n} exceeds max_str_len({self._max_str_len})") obj = self._read(n) elif 0xDC <= b <= 0xDD: size, fmt, typ = _MSGPACK_HEADERS[b] @@ -501,16 +476,14 @@ class Unpacker(object): (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_array_len: - raise ValueError( - "%s exceeds max_array_len(%s)" % (n, self._max_array_len) - ) + raise ValueError(f"{n} exceeds max_array_len({self._max_array_len})") elif 0xDE <= b <= 0xDF: size, fmt, typ = _MSGPACK_HEADERS[b] self._reserve(size) (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i) self._buff_i += size if n > self._max_map_len: - raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len)) + raise ValueError(f"{n} exceeds max_map_len({self._max_map_len})") else: raise FormatError("Unknown header: 0x%x" % b) return typ, n, obj @@ -549,17 +522,14 @@ class Unpacker(object): return if self._object_pairs_hook is not None: ret = self._object_pairs_hook( - (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT)) - for _ in range(n) + (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT)) for _ in range(n) ) else: ret = {} for _ in range(n): key = self._unpack(EX_CONSTRUCT) if self._strict_map_key and type(key) not in (str, bytes): - raise ValueError( - "%s is not allowed for map key" % str(type(key)) - ) + raise ValueError("%s is not allowed for map key" % str(type(key))) if type(key) is str: key = sys.intern(key) ret[key] = self._unpack(EX_CONSTRUCT) @@ -634,7 +604,7 @@ class Unpacker(object): return self._stream_offset -class Packer(object): +class Packer: """ MessagePack Packer @@ -844,9 +814,9 @@ class Packer(object): continue if self._datetime and check(obj, _DateTime): - raise ValueError("Cannot serialize %r where tzinfo=None" % (obj,)) + raise ValueError(f"Cannot serialize {obj!r} where tzinfo=None") - raise TypeError("Cannot serialize %r" % (obj,)) + raise TypeError(f"Cannot serialize {obj!r}") def pack(self, obj): try: @@ -933,7 +903,7 @@ class Packer(object): def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT): self._pack_map_header(n) - for (k, v) in pairs: + for k, v in pairs: self._pack(k, nest_limit - 1) self._pack(v, nest_limit - 1) diff --git a/pyproject.toml b/pyproject.toml index 195795f..86fae1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,3 +6,8 @@ requires = [ "setuptools >= 35.0.2", ] build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ["py37"] +skip_string_normalization = true diff --git a/requirements.txt b/requirements.txt index 9f3c1a0..88b5eb9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ -# Also declared in pyproject.toml, if updating here please also update there +# Also declared in pyproject.toml, if updating here please also update there. Cython~=0.29.30 -# dev only tools. no need to add pyproject -black==22.3.0 +# Tools required only for development. No need to add it to pyproject.toml file. +black==23.3.0 +pytest==7.3.1 +pyupgrade==3.3.2 diff --git a/setup.cfg b/setup.cfg index e378284..d6888fc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,7 +17,6 @@ project_urls = classifiers = Programming Language :: Python :: 3 - Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 diff --git a/setup.py b/setup.py index 15ba774..1cd1e8e 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 import io import os import sys @@ -25,7 +24,7 @@ except ImportError: def cythonize(src): - sys.stderr.write("cythonize: %r\n" % (src,)) + sys.stderr.write(f"cythonize: {src!r}\n") cython_compiler.compile([src], cplus=True) @@ -36,11 +35,7 @@ def ensure_source(src): if not have_cython: raise NoCython cythonize(pyx) - elif ( - os.path.exists(pyx) - and os.stat(src).st_mtime < os.stat(pyx).st_mtime - and have_cython - ): + elif os.path.exists(pyx) and os.stat(src).st_mtime < os.stat(pyx).st_mtime and have_cython: cythonize(pyx) return src diff --git a/test/test_buffer.py b/test/test_buffer.py index 0447058..7ee674a 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 import sys import pytest diff --git a/test/test_case.py b/test/test_case.py index 1c4e322..c4c615e 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -1,11 +1,10 @@ #!/usr/bin/env python -# coding: utf-8 from msgpack import packb, unpackb def check(length, obj, use_bin_type=True): v = packb(obj, use_bin_type=use_bin_type) - assert len(v) == length, "%r length should be %r but get %r" % (obj, length, len(v)) + assert len(v) == length, f"{obj!r} length should be {length!r} but get {len(v)!r}" assert unpackb(v, use_list=0, raw=not use_bin_type) == obj @@ -120,11 +119,11 @@ def test_match(): ), ({}, b"\x80"), ( - dict([(x, x) for x in range(15)]), + {x: x for x in range(15)}, b"\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e", ), ( - dict([(x, x) for x in range(16)]), + {x: x for x in range(16)}, b"\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f", ), ] diff --git a/test/test_except.py b/test/test_except.py index 745ebec..8c0a976 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 from pytest import raises from msgpack import packb, unpackb, Unpacker, FormatError, StackError, OutOfData diff --git a/test/test_extension.py b/test/test_extension.py index dfbe435..9e5e6aa 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,4 +1,3 @@ -from __future__ import print_function import array import msgpack from msgpack import ExtType @@ -17,9 +16,7 @@ def test_pack_ext_type(): assert p(b"A" * 16) == b"\xd8\x42" + b"A" * 16 # fixext 16 assert p(b"ABC") == b"\xc7\x03\x42ABC" # ext 8 assert p(b"A" * 0x0123) == b"\xc8\x01\x23\x42" + b"A" * 0x0123 # ext 16 - assert ( - p(b"A" * 0x00012345) == b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345 - ) # ext 32 + assert p(b"A" * 0x00012345) == b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345 # ext 32 def test_unpack_ext_type(): @@ -49,7 +46,7 @@ def test_extension_type(): except AttributeError: data = obj.tostring() return ExtType(typecode, data) - raise TypeError("Unknown type object %r" % (obj,)) + raise TypeError(f"Unknown type object {obj!r}") def ext_hook(code, data): print("ext_hook called", code, data) diff --git a/test/test_format.py b/test/test_format.py index fbbc3f9..c06c87d 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 from msgpack import unpackb @@ -25,9 +24,7 @@ def testFixRaw(): def testFixMap(): - check( - b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", {False: {None: None}, True: {None: {}}} - ) + check(b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", {False: {None: None}, True: {None: {}}}) def testUnsignedInt(): diff --git a/test/test_limits.py b/test/test_limits.py index 4314c2c..533bc11 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -1,6 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 -from __future__ import absolute_import, division, print_function, unicode_literals import pytest from msgpack import ( diff --git a/test/test_memoryview.py b/test/test_memoryview.py index 63beab1..eaadef7 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 import pytest from array import array diff --git a/test/test_newspec.py b/test/test_newspec.py index b7da486..a6f4251 100644 --- a/test/test_newspec.py +++ b/test/test_newspec.py @@ -1,5 +1,3 @@ -# coding: utf-8 - from msgpack import packb, unpackb, ExtType diff --git a/test/test_obj.py b/test/test_obj.py index 86c557c..d3f870d 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 from pytest import raises from msgpack import packb, unpackb diff --git a/test/test_pack.py b/test/test_pack.py index 65c9cb1..2753e46 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -1,6 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 -from __future__ import absolute_import, division, print_function, unicode_literals from collections import OrderedDict from io import BytesIO @@ -81,9 +79,7 @@ def testPackByteArrays(): def testIgnoreUnicodeErrors(): - re = unpackb( - packb(b"abc\xeddef", use_bin_type=False), raw=False, unicode_errors="ignore" - ) + re = unpackb(packb(b"abc\xeddef", use_bin_type=False), raw=False, unicode_errors="ignore") assert re == "abcdef" @@ -108,8 +104,8 @@ def testDecodeBinary(): def testPackFloat(): - assert packb(1.0, use_single_float=True) == b"\xca" + struct.pack(str(">f"), 1.0) - assert packb(1.0, use_single_float=False) == b"\xcb" + struct.pack(str(">d"), 1.0) + assert packb(1.0, use_single_float=True) == b"\xca" + struct.pack(">f", 1.0) + assert packb(1.0, use_single_float=False) == b"\xcb" + struct.pack(">d", 1.0) def testArraySize(sizes=[0, 5, 50, 1000]): @@ -154,7 +150,7 @@ def testMapSize(sizes=[0, 5, 50, 1000]): bio.seek(0) unpacker = Unpacker(bio, strict_map_key=False) for size in sizes: - assert unpacker.unpack() == dict((i, i * 2) for i in range(size)) + assert unpacker.unpack() == {i: i * 2 for i in range(size)} def test_odict(): diff --git a/test/test_seq.py b/test/test_seq.py index 0d5d806..def6630 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 import io import msgpack diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index c091076..6b138aa 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 import io from msgpack import Unpacker, BufferFull from msgpack import pack, packb diff --git a/test/test_stricttype.py b/test/test_stricttype.py index fe9ec6c..9ffaff2 100644 --- a/test/test_stricttype.py +++ b/test/test_stricttype.py @@ -1,5 +1,3 @@ -# coding: utf-8 - from collections import namedtuple from msgpack import packb, unpackb, ExtType @@ -10,7 +8,7 @@ def test_namedtuple(): def default(o): if isinstance(o, T): return dict(o._asdict()) - raise TypeError("Unsupported type %s" % (type(o),)) + raise TypeError(f"Unsupported type {type(o)}") packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default) unpacked = unpackb(packed, raw=False) @@ -23,7 +21,7 @@ def test_tuple(): def default(o): if isinstance(o, tuple): return {"__type__": "tuple", "value": list(o)} - raise TypeError("Unsupported type %s" % (type(o),)) + raise TypeError(f"Unsupported type {type(o)}") def convert(o): if o.get("__type__") == "tuple": @@ -44,9 +42,7 @@ def test_tuple_ext(): def default(o): if isinstance(o, tuple): # Convert to list and pack - payload = packb( - list(o), strict_types=True, use_bin_type=True, default=default - ) + payload = packb(list(o), strict_types=True, use_bin_type=True, default=default) return ExtType(MSGPACK_EXT_TYPE_TUPLE, payload) raise TypeError(repr(o)) @@ -54,7 +50,7 @@ def test_tuple_ext(): if code == MSGPACK_EXT_TYPE_TUPLE: # Unpack and convert to tuple return tuple(unpackb(payload, raw=False, ext_hook=convert)) - raise ValueError("Unknown Ext code {}".format(code)) + raise ValueError(f"Unknown Ext code {code}") data = packb(t, strict_types=True, use_bin_type=True, default=default) expected = unpackb(data, raw=False, ext_hook=convert) diff --git a/test/test_subtype.py b/test/test_subtype.py index d91d455..d5a9adb 100644 --- a/test/test_subtype.py +++ b/test/test_subtype.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 from msgpack import packb, unpackb from collections import namedtuple diff --git a/test/test_unpack.py b/test/test_unpack.py index c714102..bf3f960 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -52,7 +52,7 @@ def test_unpacker_hook_refcnt(): def test_unpacker_ext_hook(): class MyUnpacker(Unpacker): def __init__(self): - super(MyUnpacker, self).__init__(ext_hook=self._hook, raw=False) + super().__init__(ext_hook=self._hook, raw=False) def _hook(self, code, data): if code == 1: From e5249f877c18c88a0009f21097d7e48819579e60 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 21 Jul 2023 02:53:58 +0900 Subject: [PATCH 1118/1172] ci: add Python 3.12 and drop 3.7 --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8878102..76fcf27 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.11", "3.10", "3.9", "3.8", "3.7"] + py: ["3.12", "3.11", "3.10", "3.9", "3.8"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -23,6 +23,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} + allow-prereleases: true cache: "pip" - name: Build From 427736bbcc5553b0754616b58154ed26733103b6 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 21 Jul 2023 11:11:04 +0900 Subject: [PATCH 1119/1172] try Cython 3.0 (#548) --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 86fae1c..dc8bbee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ # Also declared in requirements.txt, if updating here please also update # there - "Cython~=0.29.30", + "Cython~=3.0.0", "setuptools >= 35.0.2", ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index 88b5eb9..e27df0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Also declared in pyproject.toml, if updating here please also update there. -Cython~=0.29.30 +Cython~=3.0.0 # Tools required only for development. No need to add it to pyproject.toml file. black==23.3.0 From 7cfced51501b8e0786da5d6b499331ef2f492b29 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 9 Aug 2023 18:09:42 +0900 Subject: [PATCH 1120/1172] start v1.0.6 development --- msgpack/__init__.py | 2 +- setup.cfg | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 638236f..2540120 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -6,7 +6,7 @@ import sys version = (1, 0, 5) -__version__ = "1.0.5" +__version__ = "1.0.6dev1" if os.environ.get("MSGPACK_PUREPYTHON"): diff --git a/setup.cfg b/setup.cfg index d6888fc..c2e1672 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [metadata] name = msgpack -#version = attr: msgpack.__version__ -version = attr: msgpack.version +version = attr: msgpack.__version__ +#version = attr: msgpack.version license = Apache 2.0 author = Inada Naoki author_email = songofacandy@gmail.com @@ -17,11 +17,11 @@ project_urls = classifiers = Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy Intended Audience :: Developers From 715126c67b2339381f5ad02f45d8fe367400c749 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 9 Aug 2023 18:20:05 +0900 Subject: [PATCH 1121/1172] CI: update cibuildwheel to v2.15.0 (#551) --- .github/workflows/wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 6cf2fe9..c1eb2ed 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -35,7 +35,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.12.0 + uses: pypa/cibuildwheel@v2.15.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" From 7b75b4f36819c77a12518929fecc09d94e82f5bd Mon Sep 17 00:00:00 2001 From: TW Date: Thu, 31 Aug 2023 05:56:24 +0200 Subject: [PATCH 1122/1172] sphinx-related work (#554) fixes #510 --- .github/workflows/docs.yaml | 32 ++++++++++++++++++++++++++++++++ docs/_static/README.txt | 1 + docs/api.rst | 8 ++++---- docs/conf.py | 2 +- msgpack/_packer.pyx | 3 ++- msgpack/_unpacker.pyx | 6 +++--- msgpack/ext.py | 4 ++-- msgpack/fallback.py | 9 +++++---- tox.ini | 9 +++++++++ 9 files changed, 59 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/docs.yaml create mode 100644 docs/_static/README.txt diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 0000000..a393c6b --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,32 @@ +name: docs + +on: ["push", "pull_request"] + +jobs: + docs: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + + runs-on: ubuntu-latest + steps: + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + architecture: 'x64' + + - name: Checkout + uses: actions/checkout@v3 + + - name: Build + shell: bash + run: | + pip install -r requirements.txt + make cython + pip install . + + - name: Sphinx Documentation Generator + run: | + pip install tox + tox -e sphinx diff --git a/docs/_static/README.txt b/docs/_static/README.txt new file mode 100644 index 0000000..1c70594 --- /dev/null +++ b/docs/_static/README.txt @@ -0,0 +1 @@ +Sphinx will copy the contents of docs/_static/ directory to the build location. diff --git a/docs/api.rst b/docs/api.rst index 93827e1..f5dfbbd 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -5,19 +5,19 @@ API reference .. autofunction:: pack -:func:`dump` is alias for :func:`pack` +``dump()`` is an alias for :func:`pack` .. autofunction:: packb -:func:`dumps` is alias for :func:`packb` +``dumps()`` is an alias for :func:`packb` .. autofunction:: unpack -:func:`load` is alias for :func:`unpack` +``load()`` is an alias for :func:`unpack` .. autofunction:: unpackb -:func:`loads` is alias for :func:`unpackb` +``loads()`` is an alias for :func:`unpackb` .. autoclass:: Packer :members: diff --git a/docs/conf.py b/docs/conf.py index 91ce77f..1c1895c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,7 +16,7 @@ import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) +#sys.path.insert(0, os.path.abspath('..')) # -- General configuration ----------------------------------------------------- diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 074b39f..3c39867 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -71,7 +71,8 @@ cdef class Packer(object): Packer's constructor has some keyword arguments: - :param callable default: + :param default: + When specified, it should be callable. Convert user type to builtin type that Packer supports. See also simplejson's document. diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index d5dc5ea..56126f4 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -217,7 +217,7 @@ cdef class Unpacker(object): :param file_like: File-like object having `.read(n)` method. - If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. + If specified, unpacker reads serialized data from it and `.feed()` is not usable. :param int read_size: Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) @@ -241,12 +241,12 @@ cdef class Unpacker(object): :param bool strict_map_key: If true (default), only str or bytes are accepted for map (dict) keys. - :param callable object_hook: + :param object_hook: When specified, it should be callable. Unpacker calls it with a dict argument after unpacking msgpack map. (See also simplejson) - :param callable object_pairs_hook: + :param object_pairs_hook: When specified, it should be callable. Unpacker calls it with a list of key-value pairs after unpacking msgpack map. (See also simplejson) diff --git a/msgpack/ext.py b/msgpack/ext.py index 9794294..f7f2d77 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -120,7 +120,7 @@ class Timestamp: """Create a Timestamp from posix timestamp in seconds. :param unix_float: Posix timestamp in seconds. - :type unix_float: int or float. + :type unix_float: int or float """ seconds = int(unix_sec // 1) nanoseconds = int((unix_sec % 1) * 10**9) @@ -154,7 +154,7 @@ class Timestamp: def to_datetime(self): """Get the timestamp as a UTC datetime. - :rtype: datetime. + :rtype: `datetime.datetime` """ utc = datetime.timezone.utc return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta(seconds=self.to_unix()) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index ac1eaf4..84b2617 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -139,7 +139,7 @@ class Unpacker: :param file_like: File-like object having `.read(n)` method. - If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. + If specified, unpacker reads serialized data from it and `.feed()` is not usable. :param int read_size: Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) @@ -163,12 +163,12 @@ class Unpacker: :param bool strict_map_key: If true (default), only str or bytes are accepted for map (dict) keys. - :param callable object_hook: + :param object_hook: When specified, it should be callable. Unpacker calls it with a dict argument after unpacking msgpack map. (See also simplejson) - :param callable object_pairs_hook: + :param object_pairs_hook: When specified, it should be callable. Unpacker calls it with a list of key-value pairs after unpacking msgpack map. (See also simplejson) @@ -616,7 +616,8 @@ class Packer: Packer's constructor has some keyword arguments: - :param callable default: + :param default: + When specified, it should be callable. Convert user type to builtin type that Packer supports. See also simplejson's document. diff --git a/tox.ini b/tox.ini index 1ef2d18..369eddc 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,7 @@ envlist = {py35,py36,py37,py38}-{c,pure}, {pypy,pypy3}-pure, py34-x86, + sphinx, isolated_build = true [testenv] @@ -27,3 +28,11 @@ commands= python -c 'import sys; print(hex(sys.maxsize))' python -c 'from msgpack import _cmsgpack' py.test + + +[testenv:sphinx] +changedir = docs +deps = + sphinx +commands = + sphinx-build -n -v -W --keep-going -b html -d {envtmpdir}/doctrees . {envtmpdir}/html From 423c6df265d0f964733b31a7e835fe91e4b8ea89 Mon Sep 17 00:00:00 2001 From: TW Date: Tue, 5 Sep 2023 03:51:04 +0200 Subject: [PATCH 1123/1172] move project metadata to pyproject.toml (#555) also: replace flake8 by ruff. --- docs/conf.py | 4 ++-- msgpack/__init__.py | 1 - msgpack/ext.py | 8 ++++---- msgpack/fallback.py | 2 +- pyproject.toml | 45 +++++++++++++++++++++++++++++++++++++++++ setup.cfg | 32 ----------------------------- setup.py | 2 -- test/test_buffer.py | 2 -- test/test_memoryview.py | 2 -- test/test_obj.py | 4 ++-- test/test_pack.py | 4 +--- test/test_seq.py | 2 +- test/test_subtype.py | 2 +- test/test_timestamp.py | 1 - 14 files changed, 57 insertions(+), 54 deletions(-) delete mode 100644 setup.cfg diff --git a/docs/conf.py b/docs/conf.py index 1c1895c..6eb472a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,11 +11,11 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. +#import os +#import sys #sys.path.insert(0, os.path.abspath('..')) # -- General configuration ----------------------------------------------------- diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 2540120..9a96c98 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -2,7 +2,6 @@ from .exceptions import * from .ext import ExtType, Timestamp import os -import sys version = (1, 0, 5) diff --git a/msgpack/ext.py b/msgpack/ext.py index f7f2d77..02c2c43 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -1,6 +1,5 @@ from collections import namedtuple import datetime -import sys import struct @@ -20,8 +19,9 @@ class ExtType(namedtuple("ExtType", "code data")): class Timestamp: """Timestamp represents the Timestamp extension type in msgpack. - When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. When using pure-Python - msgpack, :func:`to_bytes` and :func:`from_bytes` are used to pack and unpack `Timestamp`. + When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. + When using pure-Python msgpack, :func:`to_bytes` and :func:`from_bytes` are used to pack and + unpack `Timestamp`. This class is immutable: Do not override seconds and nanoseconds. """ @@ -39,7 +39,7 @@ class Timestamp: Number of nanoseconds to add to `seconds` to get fractional time. Maximum is 999_999_999. Default is 0. - Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns. + Note: Negative times (before the UNIX epoch) are represented as neg. seconds + pos. ns. """ if not isinstance(seconds, int): raise TypeError("seconds must be an integer") diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 84b2617..a174162 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -530,7 +530,7 @@ class Unpacker: key = self._unpack(EX_CONSTRUCT) if self._strict_map_key and type(key) not in (str, bytes): raise ValueError("%s is not allowed for map key" % str(type(key))) - if type(key) is str: + if isinstance(key, str): key = sys.intern(key) ret[key] = self._unpack(EX_CONSTRUCT) if self._object_hook is not None: diff --git a/pyproject.toml b/pyproject.toml index dc8bbee..f37d213 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,52 @@ requires = [ ] build-backend = "setuptools.build_meta" +[project] +name = "msgpack" +dynamic = ["version"] +license = {text="Apache 2.0"} +authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}] +description = "MessagePack serializer" +readme = "README.md" +#keywords = ["python", "msgpack", "messagepack", "serializer", "serialization", "binary"] +#requires-python = ">=3.8" +classifiers = [ +# "Development Status :: 5 - Production/Stable", +# "Operating System :: OS Independent", +# "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", +] + +[project.urls] +Homepage = "https://msgpack.org/" +Documentation = "https://msgpack-python.readthedocs.io/" +Repository = "https://github.com/msgpack/msgpack-python/" +Tracker = "https://github.com/msgpack/msgpack-python/issues" +#Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst" + +[tool.setuptools.dynamic] +version = {attr = "msgpack.__version__"} + [tool.black] line-length = 100 target-version = ["py37"] skip_string_normalization = true + +[tool.ruff] +line-length = 100 +target-version = "py38" +ignore = [] + +[tool.ruff.per-file-ignores] +"msgpack/__init__.py" = ["F401", "F403"] +"msgpack/fallback.py" = ["E731"] +"test/test_seq.py" = ["E501"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index c2e1672..0000000 --- a/setup.cfg +++ /dev/null @@ -1,32 +0,0 @@ -[metadata] -name = msgpack -version = attr: msgpack.__version__ -#version = attr: msgpack.version -license = Apache 2.0 -author = Inada Naoki -author_email = songofacandy@gmail.com -description = MessagePack serializer -long_description = file: README.md -long_description_content_type = text/markdown -url = https://msgpack.org/ - -project_urls = - Documentation = https://msgpack-python.readthedocs.io/ - Source = https://github.com/msgpack/msgpack-python - Tracker = https://github.com/msgpack/msgpack-python/issues - -classifiers = - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - Programming Language :: Python :: 3.12 - Programming Language :: Python :: Implementation :: CPython - Programming Language :: Python :: Implementation :: PyPy - Intended Audience :: Developers - License :: OSI Approved :: Apache Software License - -[flake8] -max_line_length = 100 - diff --git a/setup.py b/setup.py index 1cd1e8e..7a34c8c 100755 --- a/setup.py +++ b/setup.py @@ -1,8 +1,6 @@ #!/usr/bin/env python -import io import os import sys -from glob import glob from setuptools import setup, Extension from setuptools.command.build_ext import build_ext from setuptools.command.sdist import sdist diff --git a/test/test_buffer.py b/test/test_buffer.py index 7ee674a..a3db339 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -import sys -import pytest from msgpack import packb, unpackb diff --git a/test/test_memoryview.py b/test/test_memoryview.py index eaadef7..dc319a6 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -1,9 +1,7 @@ #!/usr/bin/env python -import pytest from array import array from msgpack import packb, unpackb -import sys def make_array(f, data): diff --git a/test/test_obj.py b/test/test_obj.py index d3f870d..f78bf42 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -33,7 +33,7 @@ def test_decode_pairs_hook(): prod_sum = 1 * 2 + 3 * 4 unpacked = unpackb( packed, - object_pairs_hook=lambda l: sum(k * v for k, v in l), + object_pairs_hook=lambda lst: sum(k * v for k, v in lst), use_list=1, strict_map_key=False, ) @@ -48,7 +48,7 @@ def test_only_one_obj_hook(): def test_bad_hook(): with raises(TypeError): packed = packb([3, 1 + 2j], default=lambda o: o) - unpacked = unpackb(packed, use_list=1) + unpackb(packed, use_list=1) def _arr_to_str(arr): diff --git a/test/test_pack.py b/test/test_pack.py index 2753e46..4232537 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -3,12 +3,10 @@ from collections import OrderedDict from io import BytesIO import struct -import sys import pytest -from pytest import raises, xfail -from msgpack import packb, unpackb, Unpacker, Packer, pack +from msgpack import packb, unpackb, Unpacker, Packer def check(data, use_list=False): diff --git a/test/test_seq.py b/test/test_seq.py index def6630..16d9dde 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -34,7 +34,7 @@ def test_exceeding_unpacker_read_size(): read_count = 0 for idx, o in enumerate(unpacker): - assert type(o) == bytes + assert isinstance(o, bytes) assert o == gen_binary_data(idx) read_count += 1 diff --git a/test/test_subtype.py b/test/test_subtype.py index d5a9adb..0d1c41a 100644 --- a/test/test_subtype.py +++ b/test/test_subtype.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from msgpack import packb, unpackb +from msgpack import packb from collections import namedtuple diff --git a/test/test_timestamp.py b/test/test_timestamp.py index af84a2f..db5cc57 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -1,5 +1,4 @@ import pytest -import sys import datetime import msgpack from msgpack.ext import Timestamp From ef15f4a62c25114bec1db91aa4006ae2d3a9fb53 Mon Sep 17 00:00:00 2001 From: TW Date: Thu, 7 Sep 2023 14:25:07 +0200 Subject: [PATCH 1124/1172] add a basic .readthedocs.yaml file (#558) --- .readthedocs.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..8b5aaf1 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,12 @@ +# Read the Docs configuration file for Sphinx projects. +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details. + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + configuration: docs/conf.py From 41d6239c0a3bfb1daabe1a45ffdbecf4e9aa5469 Mon Sep 17 00:00:00 2001 From: TW Date: Tue, 12 Sep 2023 19:51:12 +0200 Subject: [PATCH 1125/1172] fix .readthedocs.yaml, fixes #559 (#560) --- .readthedocs.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 8b5aaf1..7447895 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,6 +7,17 @@ build: os: ubuntu-22.04 tools: python: "3.11" + apt_packages: + - build-essential + jobs: + pre_install: + - pip install -r requirements.txt + - make cython + +python: + install: + - method: pip + path: . sphinx: configuration: docs/conf.py From 4e10c10aaa8350f23e4b85d27ff131f7b4fd13e2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 13 Sep 2023 18:40:04 +0900 Subject: [PATCH 1126/1172] prepare for 1.0.6rc1 (#557) --- ChangeLog.rst | 9 +++++++++ msgpack/__init__.py | 4 ++-- pyproject.toml | 12 ++++++------ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 5354799..bf345dd 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,12 @@ +1.0.6rc1 +======== + +Release Date: 2023-09-13 + +* Add Python 3.12 wheels (#517) +* Remove Python 2.7, 3.6, and 3.7 support + + 1.0.5 ===== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 9a96c98..2e20133 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .ext import ExtType, Timestamp import os -version = (1, 0, 5) -__version__ = "1.0.6dev1" +version = (1, 0, 6, "rc", 1) +__version__ = "1.0.6rc1" if os.environ.get("MSGPACK_PUREPYTHON"): diff --git a/pyproject.toml b/pyproject.toml index f37d213..a63009a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,12 +14,12 @@ license = {text="Apache 2.0"} authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}] description = "MessagePack serializer" readme = "README.md" -#keywords = ["python", "msgpack", "messagepack", "serializer", "serialization", "binary"] -#requires-python = ">=3.8" +keywords = ["msgpack", "messagepack", "serializer", "serialization", "binary"] +requires-python = ">=3.8" classifiers = [ -# "Development Status :: 5 - Production/Stable", -# "Operating System :: OS Independent", -# "Programming Language :: Python", + "Development Status :: 5 - Production/Stable", + "Operating System :: OS Independent", + "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", @@ -37,7 +37,7 @@ Homepage = "https://msgpack.org/" Documentation = "https://msgpack-python.readthedocs.io/" Repository = "https://github.com/msgpack/msgpack-python/" Tracker = "https://github.com/msgpack/msgpack-python/issues" -#Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst" +Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst" [tool.setuptools.dynamic] version = {attr = "msgpack.__version__"} From e1d3d5d5c386b8b2fa99c812b4648f6532cab032 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 15 Sep 2023 12:02:06 +0900 Subject: [PATCH 1127/1172] update actions (#563) --- .github/workflows/black.yaml | 2 +- .github/workflows/docs.yaml | 2 +- .github/workflows/test.yml | 2 +- .github/workflows/wheel.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index 1e28b7b..e091792 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -17,7 +17,7 @@ jobs: architecture: 'x64' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Black Code Formatter run: | diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index a393c6b..80bbba7 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -17,7 +17,7 @@ jobs: architecture: 'x64' - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build shell: bash diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 76fcf27..4eb8849 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index c1eb2ed..0412a38 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -14,11 +14,11 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 From b1b0edaeedd073f21023ec01a60bfa9da077ad2b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 21 Sep 2023 14:58:37 +0900 Subject: [PATCH 1128/1172] release v1.0.6 (#564) --- ChangeLog.rst | 6 +++--- msgpack/__init__.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index bf345dd..bad51aa 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,7 +1,7 @@ -1.0.6rc1 -======== +1.0.6 +===== -Release Date: 2023-09-13 +Release Date: 2023-09-21 * Add Python 3.12 wheels (#517) * Remove Python 2.7, 3.6, and 3.7 support diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 2e20133..781bcdf 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .ext import ExtType, Timestamp import os -version = (1, 0, 6, "rc", 1) -__version__ = "1.0.6rc1" +version = (1, 0, 6) +__version__ = "1.0.6" if os.environ.get("MSGPACK_PUREPYTHON"): From ecf03748c7241a0fb6bef733c7e5d2d68179b670 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 28 Sep 2023 15:03:16 +0900 Subject: [PATCH 1129/1172] remove inline macro for msvc (#567) --- msgpack/pack.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/msgpack/pack.h b/msgpack/pack.h index 1e849ac..2453428 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -26,10 +26,6 @@ extern "C" { #endif -#ifdef _MSC_VER -#define inline __inline -#endif - typedef struct msgpack_packer { char *buf; size_t length; From acd068439233b8f04543c4ee81c18c8dbb681aba Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 28 Sep 2023 15:25:10 +0900 Subject: [PATCH 1130/1172] do not fallback on build error (#568) --- setup.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) mode change 100755 => 100644 setup.py diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index 7a34c8c..a13bd81 --- a/setup.py +++ b/setup.py @@ -22,6 +22,8 @@ except ImportError: def cythonize(src): + if not have_cython: + raise Exception("Cython is required for building from checkout") sys.stderr.write(f"cythonize: {src!r}\n") cython_compiler.compile([src], cplus=True) @@ -29,31 +31,15 @@ def cythonize(src): def ensure_source(src): pyx = os.path.splitext(src)[0] + ".pyx" - if not os.path.exists(src): - if not have_cython: - raise NoCython + if not os.path.exists(src) or have_cython and os.stat(src).st_mtime < os.stat(pyx).st_mtime: cythonize(pyx) - elif os.path.exists(pyx) and os.stat(src).st_mtime < os.stat(pyx).st_mtime and have_cython: - cythonize(pyx) - return src class BuildExt(build_ext): def build_extension(self, ext): - try: - ext.sources = list(map(ensure_source, ext.sources)) - except NoCython: - print("WARNING") - print("Cython is required for building extension from checkout.") - print("Install Cython >= 0.16 or install msgpack from PyPI.") - print("Falling back to pure Python implementation.") - return - try: - return build_ext.build_extension(self, ext) - except Exception as e: - print("WARNING: Failed to compile extension modules.") - print("msgpack uses fallback pure python implementation.") - print(e) + for src in ext.sources: + ensure_source(src) + return build_ext.build_extension(self, ext) # Cython is required for sdist From 2982e9ff729eae150d67ee608fdf1d01d93d8e3f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 28 Sep 2023 17:31:52 +0900 Subject: [PATCH 1131/1172] release v1.0.7 (#569) --- .gitignore | 1 + ChangeLog.rst | 13 +++++++++++++ msgpack/__init__.py | 4 ++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 800f1c2..8a06e26 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ MANIFEST build/* dist/* .tox +.python-version *.pyc *.pyo *.so diff --git a/ChangeLog.rst b/ChangeLog.rst index bad51aa..ca74ebe 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,8 +1,21 @@ +1.0.7 +===== + +Release Date: 2023-09-28 + +* Fix build error of extension module on Windows. (#567) +* ``setup.py`` doesn't skip build error of extension module. (#568) + + 1.0.6 ===== Release Date: 2023-09-21 +.. note:: + v1.0.6 Wheels for Windows don't contain extension module. + Please upgrade to v1.0.7 or newer. + * Add Python 3.12 wheels (#517) * Remove Python 2.7, 3.6, and 3.7 support diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 781bcdf..60a088c 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .ext import ExtType, Timestamp import os -version = (1, 0, 6) -__version__ = "1.0.6" +version = (1, 0, 7) +__version__ = "1.0.7" if os.environ.get("MSGPACK_PUREPYTHON"): From c78026102c981d734c6d8fec6b2790ee31212f27 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 23:34:32 +0900 Subject: [PATCH 1132/1172] doc: use sphinx-rtd-theme (#575) --- .github/workflows/docs.yaml | 1 - .readthedocs.yaml | 1 + docs/conf.py | 4 +--- docs/requirements.txt | 2 ++ tox.ini | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 docs/requirements.txt diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 80bbba7..8c8298a 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -24,7 +24,6 @@ jobs: run: | pip install -r requirements.txt make cython - pip install . - name: Sphinx Documentation Generator run: | diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 7447895..88d8718 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -18,6 +18,7 @@ python: install: - method: pip path: . + - requirements: docs/requirements.txt sphinx: configuration: docs/conf.py diff --git a/docs/conf.py b/docs/conf.py index 6eb472a..ab0ad3c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # msgpack documentation build configuration file, created by # sphinx-quickstart on Sun Feb 24 14:20:50 2013. # @@ -91,7 +89,7 @@ pygments_style = "sphinx" # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = "sphinxdoc" +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..8d45d0b --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx~=7.2 +sphinx-rtd-theme~=1.3.0 diff --git a/tox.ini b/tox.ini index 369eddc..49364be 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,6 @@ commands= [testenv:sphinx] changedir = docs deps = - sphinx + -r docs/requirements.txt commands = sphinx-build -n -v -W --keep-going -b html -d {envtmpdir}/doctrees . {envtmpdir}/html From 140864249fd0f67dffaeceeb168ffe9cdf6f1964 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 20 Dec 2023 20:46:04 +0900 Subject: [PATCH 1133/1172] exclude C/Cython files from wheel (#577) --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a63009a..121b1fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,10 @@ Repository = "https://github.com/msgpack/msgpack-python/" Tracker = "https://github.com/msgpack/msgpack-python/issues" Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst" +[tool.setuptools] +# Do not install C/C++/Cython source files +include-package-data = false + [tool.setuptools.dynamic] version = {attr = "msgpack.__version__"} From 039022cecb04b62a29afb8260b81f57a937aaaaa Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 1 Mar 2024 19:24:06 +0900 Subject: [PATCH 1134/1172] update Cython (#581) --- .github/workflows/wheel.yml | 10 +++++++++- pyproject.toml | 2 +- requirements.txt | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 0412a38..4a8847d 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -3,6 +3,7 @@ on: push: branches: [main] create: + workflow_dispatch: jobs: build_wheels: @@ -23,7 +24,7 @@ jobs: platforms: arm64 - name: Set up Python 3.x - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.x" cache: "pip" @@ -43,6 +44,13 @@ jobs: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_SKIP: pp* + - name: Build pure Python wheel + env: + MSGPACK_PUREPYTHON: "1" + run: | + pip install build + python -m build -w -o wheelhouse + - name: Upload Wheels to artifact uses: actions/upload-artifact@v1 with: diff --git a/pyproject.toml b/pyproject.toml index 121b1fd..f9af967 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ requires = [ # Also declared in requirements.txt, if updating here please also update # there - "Cython~=3.0.0", + "Cython~=3.0.8", "setuptools >= 35.0.2", ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index e27df0f..839dc5f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Also declared in pyproject.toml, if updating here please also update there. -Cython~=3.0.0 +Cython~=3.0.8 # Tools required only for development. No need to add it to pyproject.toml file. black==23.3.0 From bf7bf88ad0b3cd7a8cd74e8251521fde743e9af9 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 1 Mar 2024 20:09:55 +0900 Subject: [PATCH 1135/1172] ci: update workflows (#582) --- .github/workflows/test.yml | 14 +++++++++++++- .github/workflows/wheel.yml | 6 +++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4eb8849..1faeb0c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.py }} allow-prereleases: true @@ -42,3 +42,15 @@ jobs: shell: bash run: | MSGPACK_PUREPYTHON=1 pytest -v test + + - name: build packages + shell: bash + run: | + pip install build + python -m build + + - name: upload packages + uses: actions/upload-artifact@v4 + with: + name: dist-${{ matrix.os }}-${{ matrix.py }} + path: dist diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 4a8847d..e91325b 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -36,7 +36,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.15.0 + uses: pypa/cibuildwheel@v2.16.5 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" @@ -52,7 +52,7 @@ jobs: python -m build -w -o wheelhouse - name: Upload Wheels to artifact - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: - name: Wheels + name: wheels-${{ matrix.os }} path: wheelhouse From 9aedf8ed7f632044d42984e9710fefbd97023f71 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 1 Mar 2024 20:35:28 +0900 Subject: [PATCH 1136/1172] Release v1.0.8 (#583) --- ChangeLog.rst | 10 ++++++++++ msgpack/__init__.py | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index ca74ebe..2408bc9 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,13 @@ +1.0.8 +===== + +Release Date: 2024-03-01 + +* Update Cython to 3.0.8. This fixes memory leak when iterating + ``Unpacker`` object on Python 3.12. +* Do not include C/Cython files in binary wheels. + + 1.0.7 ===== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 60a088c..919b86f 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .ext import ExtType, Timestamp import os -version = (1, 0, 7) -__version__ = "1.0.7" +version = (1, 0, 8) +__version__ = "1.0.8" if os.environ.get("MSGPACK_PUREPYTHON"): From e77672200bf89f2093b96a4abd0e4eeb253975b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?hakan=20aky=C3=BCrek?= Date: Sat, 20 Apr 2024 00:46:30 +0200 Subject: [PATCH 1137/1172] Avoid using floating points during timestamp-datetime conversions (#591) --- msgpack/ext.py | 6 ++++-- test/test_timestamp.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/msgpack/ext.py b/msgpack/ext.py index 02c2c43..3940fe0 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -157,7 +157,9 @@ class Timestamp: :rtype: `datetime.datetime` """ utc = datetime.timezone.utc - return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta(seconds=self.to_unix()) + return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta( + seconds=self.seconds, microseconds=self.nanoseconds // 1000 + ) @staticmethod def from_datetime(dt): @@ -165,4 +167,4 @@ class Timestamp: :rtype: Timestamp """ - return Timestamp.from_unix(dt.timestamp()) + return Timestamp(seconds=int(dt.timestamp()), nanoseconds=dt.microsecond * 1000) diff --git a/test/test_timestamp.py b/test/test_timestamp.py index db5cc57..f9bc835 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -86,6 +86,21 @@ def test_timestamp_datetime(): utc = datetime.timezone.utc assert t.to_datetime() == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=utc) + ts = datetime.datetime(2024, 4, 16, 8, 43, 9, 420317, tzinfo=utc) + ts2 = datetime.datetime(2024, 4, 16, 8, 43, 9, 420318, tzinfo=utc) + + assert ( + Timestamp.from_datetime(ts2).nanoseconds - Timestamp.from_datetime(ts).nanoseconds == 1000 + ) + + ts3 = datetime.datetime(2024, 4, 16, 8, 43, 9, 4256) + ts4 = datetime.datetime(2024, 4, 16, 8, 43, 9, 4257) + assert ( + Timestamp.from_datetime(ts4).nanoseconds - Timestamp.from_datetime(ts3).nanoseconds == 1000 + ) + + assert Timestamp.from_datetime(ts).to_datetime() == ts + def test_unpack_datetime(): t = Timestamp(42, 14) From 2eca765533106cc4bbb44ecc565b1034ac50f91b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 May 2024 15:17:54 +0900 Subject: [PATCH 1138/1172] use ruff instead of black (#598) --- .github/workflows/docs.yaml | 21 +++++++----- .github/workflows/{black.yaml => lint.yaml} | 19 +++++------ Makefile | 14 ++++++-- docs/requirements.txt | 4 +-- msgpack/fallback.py | 1 + pyproject.toml | 4 +-- requirements.txt | 7 +--- test/test_memoryview.py | 2 +- test/test_pack.py | 2 +- test/test_read_size.py | 1 + tox.ini | 38 --------------------- 11 files changed, 40 insertions(+), 73 deletions(-) rename .github/workflows/{black.yaml => lint.yaml} (56%) delete mode 100644 tox.ini diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 8c8298a..08fc2f4 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -10,22 +10,25 @@ jobs: runs-on: ubuntu-latest steps: - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.x' - architecture: 'x64' - - name: Checkout uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + architecture: 'x64' + cache: "pip" + cache-dependency-path: | + requirements.txt + docs/requirements.txt + - name: Build - shell: bash run: | pip install -r requirements.txt make cython - name: Sphinx Documentation Generator run: | - pip install tox - tox -e sphinx + pip install -r docs/requirements.txt + make docs diff --git a/.github/workflows/black.yaml b/.github/workflows/lint.yaml similarity index 56% rename from .github/workflows/black.yaml rename to .github/workflows/lint.yaml index e091792..198cf7b 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/lint.yaml @@ -1,25 +1,22 @@ -name: Black +name: lint on: ["push", "pull_request"] jobs: - black: + lint: # We want to run on external PRs, but not on our own internal PRs as they'll be run # by the push to the branch. if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository runs-on: ubuntu-latest steps: - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.x' - architecture: 'x64' - - name: Checkout uses: actions/checkout@v4 - - name: Black Code Formatter + - name: ruff check run: | - pip install black==22.3.0 - black -S --diff --check msgpack/ test/ setup.py + pipx run ruff check --diff msgpack/ test/ setup.py + + - name: ruff format + run: | + pipx run ruff format --diff msgpack/ test/ setup.py diff --git a/Makefile b/Makefile index e4f22da..5c1863c 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,17 @@ PYTHON_SOURCES = msgpack test setup.py all: cython python setup.py build_ext -i -f -.PHONY: black -black: - black $(PYTHON_SOURCES) +.PHONY: format +format: + pipx run ruff format $(PYTHON_SOURCES) + +.PHONY: lint +lint: + pipx run ruff check $(PYTHON_SOURCES) + +.PHONY: doc +doc: + cd docs && sphinx-build -n -v -W --keep-going -b html -d doctrees . html .PHONY: pyupgrade pyupgrade: diff --git a/docs/requirements.txt b/docs/requirements.txt index 8d45d0b..26002de 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -sphinx~=7.2 -sphinx-rtd-theme~=1.3.0 +sphinx~=7.3.7 +sphinx-rtd-theme~=2.0.0 diff --git a/msgpack/fallback.py b/msgpack/fallback.py index a174162..ea4c4ce 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -1,4 +1,5 @@ """Fallback pure Python implementation of msgpack""" + from datetime import datetime as _DateTime import sys import struct diff --git a/pyproject.toml b/pyproject.toml index f9af967..d99375a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,9 +54,9 @@ skip_string_normalization = true [tool.ruff] line-length = 100 target-version = "py38" -ignore = [] +lint.ignore = [] -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "msgpack/__init__.py" = ["F401", "F403"] "msgpack/fallback.py" = ["E731"] "test/test_seq.py" = ["E501"] diff --git a/requirements.txt b/requirements.txt index 839dc5f..1164a94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,2 @@ # Also declared in pyproject.toml, if updating here please also update there. -Cython~=3.0.8 - -# Tools required only for development. No need to add it to pyproject.toml file. -black==23.3.0 -pytest==7.3.1 -pyupgrade==3.3.2 +Cython~=3.0.10 diff --git a/test/test_memoryview.py b/test/test_memoryview.py index dc319a6..eff4bca 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -95,4 +95,4 @@ def test_multidim_memoryview(): view = memoryview(b"\00" * 6) data = view.cast(view.format, (3, 2)) packed = packb(data) - assert packed == b'\xc4\x06\x00\x00\x00\x00\x00\x00' + assert packed == b"\xc4\x06\x00\x00\x00\x00\x00\x00" diff --git a/test/test_pack.py b/test/test_pack.py index 4232537..4a0ef40 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -89,7 +89,7 @@ def testStrictUnicodeUnpack(): def testIgnoreErrorsPack(): re = unpackb( - packb("abc\uDC80\uDCFFdef", use_bin_type=True, unicode_errors="ignore"), + packb("abc\udc80\udcffdef", use_bin_type=True, unicode_errors="ignore"), raw=False, use_list=1, ) diff --git a/test/test_read_size.py b/test/test_read_size.py index 33a7e7d..a7d61fd 100644 --- a/test/test_read_size.py +++ b/test/test_read_size.py @@ -1,4 +1,5 @@ """Test Unpacker's read_array_header and read_map_header methods""" + from msgpack import packb, Unpacker, OutOfData UnexpectedTypeException = ValueError diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 49364be..0000000 --- a/tox.ini +++ /dev/null @@ -1,38 +0,0 @@ -[tox] -envlist = - {py35,py36,py37,py38}-{c,pure}, - {pypy,pypy3}-pure, - py34-x86, - sphinx, -isolated_build = true - -[testenv] -deps= - pytest - -changedir=test -commands= - c,x86: python -c 'from msgpack import _cmsgpack' - c,x86: py.test - pure: py.test -setenv= - pure: MSGPACK_PUREPYTHON=x - -[testenv:py34-x86] -basepython=python3.4-x86 -deps= - pytest - -changedir=test -commands= - python -c 'import sys; print(hex(sys.maxsize))' - python -c 'from msgpack import _cmsgpack' - py.test - - -[testenv:sphinx] -changedir = docs -deps = - -r docs/requirements.txt -commands = - sphinx-build -n -v -W --keep-going -b html -d {envtmpdir}/doctrees . {envtmpdir}/html From 0602baf3ea7fb597d1c78a90980071e03a536836 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 May 2024 18:20:09 +0900 Subject: [PATCH 1139/1172] update Cython and setuptools (#599) --- DEVELOP.md | 8 -------- pyproject.toml | 7 +++---- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/DEVELOP.md b/DEVELOP.md index 9c823c3..27adf8c 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -1,13 +1,5 @@ # Developer's note -## Wheels - -Wheels for macOS and Linux are built on Travis and AppVeyr, in -[methane/msgpack-wheels](https://github.com/methane/msgpack-wheels) repository. - -Wheels for Windows are built on Github Actions in this repository. - - ### Build ``` diff --git a/pyproject.toml b/pyproject.toml index d99375a..6254f06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,8 @@ [build-system] requires = [ - # Also declared in requirements.txt, if updating here please also update - # there - "Cython~=3.0.8", - "setuptools >= 35.0.2", + # Also declared in requirements.txt, if updating here please also update there + "Cython~=3.0.10", + "setuptools >= 69.5.1", ] build-backend = "setuptools.build_meta" From 3e9a2a7419714c294be0590aab24f2dc040581f5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 16:01:48 +0900 Subject: [PATCH 1140/1172] Stop using c++ (#600) Python 3.13a6+ & C++ & Cython cause compile error on some compilers. --- .github/workflows/test.yml | 2 +- Makefile | 2 +- msgpack/_unpacker.pyx | 2 +- msgpack/pack.h | 2 + msgpack/unpack_container_header.h | 51 +++++++++++++++++++ msgpack/unpack_template.h | 83 ++++++++----------------------- setup.py | 6 +-- 7 files changed, 80 insertions(+), 68 deletions(-) create mode 100644 msgpack/unpack_container_header.h diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1faeb0c..530238c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.13-dev", "3.12", "3.11", "3.10", "3.9", "3.8"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} diff --git a/Makefile b/Makefile index 5c1863c..3ce178f 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ pyupgrade: .PHONY: cython cython: - cython --cplus msgpack/_cmsgpack.pyx + cython msgpack/_cmsgpack.pyx .PHONY: test test: cython diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 56126f4..2771e7b 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -35,7 +35,7 @@ cdef extern from "unpack.h": PyObject* timestamp_t PyObject *giga; PyObject *utc; - char *unicode_errors + const char *unicode_errors Py_ssize_t max_str_len Py_ssize_t max_bin_len Py_ssize_t max_array_len diff --git a/msgpack/pack.h b/msgpack/pack.h index 2453428..901fad7 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -24,6 +24,8 @@ #ifdef __cplusplus extern "C" { +#else +#define bool char #endif typedef struct msgpack_packer { diff --git a/msgpack/unpack_container_header.h b/msgpack/unpack_container_header.h new file mode 100644 index 0000000..c14a3c2 --- /dev/null +++ b/msgpack/unpack_container_header.h @@ -0,0 +1,51 @@ +static inline int unpack_container_header(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) +{ + assert(len >= *off); + uint32_t size; + const unsigned char *const p = (unsigned char*)data + *off; + +#define inc_offset(inc) \ + if (len - *off < inc) \ + return 0; \ + *off += inc; + + switch (*p) { + case var_offset: + inc_offset(3); + size = _msgpack_load16(uint16_t, p + 1); + break; + case var_offset + 1: + inc_offset(5); + size = _msgpack_load32(uint32_t, p + 1); + break; +#ifdef USE_CASE_RANGE + case fixed_offset + 0x0 ... fixed_offset + 0xf: +#else + case fixed_offset + 0x0: + case fixed_offset + 0x1: + case fixed_offset + 0x2: + case fixed_offset + 0x3: + case fixed_offset + 0x4: + case fixed_offset + 0x5: + case fixed_offset + 0x6: + case fixed_offset + 0x7: + case fixed_offset + 0x8: + case fixed_offset + 0x9: + case fixed_offset + 0xa: + case fixed_offset + 0xb: + case fixed_offset + 0xc: + case fixed_offset + 0xd: + case fixed_offset + 0xe: + case fixed_offset + 0xf: +#endif + ++*off; + size = ((unsigned int)*p) & 0x0f; + break; + default: + PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); + return -1; + } + unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj); + return 1; +} + diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 8b9fcc1..cce29e7 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -75,8 +75,7 @@ static inline void unpack_clear(unpack_context *ctx) Py_CLEAR(ctx->stack[0].obj); } -template -static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) +static inline int unpack_execute(bool construct, unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) { assert(len >= *off); @@ -386,6 +385,7 @@ _end: #undef construct_cb } +#undef NEXT_CS #undef SWITCH_RANGE_BEGIN #undef SWITCH_RANGE #undef SWITCH_RANGE_DEFAULT @@ -397,68 +397,27 @@ _end: #undef again_fixed_trail_if_zero #undef start_container -template -static inline int unpack_container_header(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) -{ - assert(len >= *off); - uint32_t size; - const unsigned char *const p = (unsigned char*)data + *off; - -#define inc_offset(inc) \ - if (len - *off < inc) \ - return 0; \ - *off += inc; - - switch (*p) { - case var_offset: - inc_offset(3); - size = _msgpack_load16(uint16_t, p + 1); - break; - case var_offset + 1: - inc_offset(5); - size = _msgpack_load32(uint32_t, p + 1); - break; -#ifdef USE_CASE_RANGE - case fixed_offset + 0x0 ... fixed_offset + 0xf: -#else - case fixed_offset + 0x0: - case fixed_offset + 0x1: - case fixed_offset + 0x2: - case fixed_offset + 0x3: - case fixed_offset + 0x4: - case fixed_offset + 0x5: - case fixed_offset + 0x6: - case fixed_offset + 0x7: - case fixed_offset + 0x8: - case fixed_offset + 0x9: - case fixed_offset + 0xa: - case fixed_offset + 0xb: - case fixed_offset + 0xc: - case fixed_offset + 0xd: - case fixed_offset + 0xe: - case fixed_offset + 0xf: -#endif - ++*off; - size = ((unsigned int)*p) & 0x0f; - break; - default: - PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); - return -1; - } - unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj); - return 1; +static int unpack_construct(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) { + return unpack_execute(1, ctx, data, len, off); +} +static int unpack_skip(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) { + return unpack_execute(0, ctx, data, len, off); } -#undef SWITCH_RANGE_BEGIN -#undef SWITCH_RANGE -#undef SWITCH_RANGE_DEFAULT -#undef SWITCH_RANGE_END +#define unpack_container_header read_array_header +#define fixed_offset 0x90 +#define var_offset 0xdc +#include "unpack_container_header.h" +#undef unpack_container_header +#undef fixed_offset +#undef var_offset -static const execute_fn unpack_construct = &unpack_execute; -static const execute_fn unpack_skip = &unpack_execute; -static const execute_fn read_array_header = &unpack_container_header<0x90, 0xdc>; -static const execute_fn read_map_header = &unpack_container_header<0x80, 0xde>; - -#undef NEXT_CS +#define unpack_container_header read_map_header +#define fixed_offset 0x80 +#define var_offset 0xde +#include "unpack_container_header.h" +#undef unpack_container_header +#undef fixed_offset +#undef var_offset /* vim: set ts=4 sw=4 sts=4 expandtab */ diff --git a/setup.py b/setup.py index a13bd81..dc14a26 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def cythonize(src): if not have_cython: raise Exception("Cython is required for building from checkout") sys.stderr.write(f"cythonize: {src!r}\n") - cython_compiler.compile([src], cplus=True) + cython_compiler.compile([src]) def ensure_source(src): @@ -51,17 +51,17 @@ class Sdist(sdist): libraries = [] macros = [] +ext_modules = [] if sys.platform == "win32": libraries.append("ws2_32") macros = [("__LITTLE_ENDIAN__", "1")] -ext_modules = [] if not PYPY and not os.environ.get("MSGPACK_PUREPYTHON"): ext_modules.append( Extension( "msgpack._cmsgpack", - sources=["msgpack/_cmsgpack.cpp"], + sources=["msgpack/_cmsgpack.c"], libraries=libraries, include_dirs=["."], define_macros=macros, From b389ccf2f72355e23836ced193b555401508ef81 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 16:10:37 +0900 Subject: [PATCH 1141/1172] update README (#561) --- README.md | 116 +++++++++++++++++++++++------------------------------- 1 file changed, 50 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 61f99e1..d3247bd 100644 --- a/README.md +++ b/README.md @@ -10,53 +10,6 @@ It lets you exchange data among multiple languages like JSON. But it's faster and smaller. This package provides CPython bindings for reading and writing MessagePack data. - -## Very important notes for existing users - -### PyPI package name - -Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5. - -When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before -`pip install -U msgpack`. - - -### Compatibility with the old format - -You can use `use_bin_type=False` option to pack `bytes` -object into raw type in the old msgpack spec, instead of bin type in new msgpack spec. - -You can unpack old msgpack format using `raw=True` option. -It unpacks str (raw) type in msgpack into Python bytes. - -See note below for detail. - - -### Major breaking changes in msgpack 1.0 - -* Python 2 - - * The extension module does not support Python 2 anymore. - The pure Python implementation (`msgpack.fallback`) is used for Python 2. - -* Packer - - * `use_bin_type=True` by default. bytes are encoded in bin type in msgpack. - **If you are still using Python 2, you must use unicode for all string types.** - You can use `use_bin_type=False` to encode into old msgpack format. - * `encoding` option is removed. UTF-8 is used always. - -* Unpacker - - * `raw=False` by default. It assumes str types are valid UTF-8 string - and decode them to Python str (unicode) object. - * `encoding` option is removed. You can use `raw=True` to support old format. - * Default value of `max_buffer_size` is changed from 0 to 100 MiB. - * Default value of `strict_map_key` is changed to True to avoid hashdos. - You need to pass `strict_map_key=False` if you have data which contain map keys - which type is not bytes or str. - - ## Install ``` @@ -65,12 +18,9 @@ $ pip install msgpack ### Pure Python implementation -The extension module in msgpack (`msgpack._cmsgpack`) does not support -Python 2 and PyPy. - -But msgpack provides a pure Python implementation (`msgpack.fallback`) -for PyPy and Python 2. +The extension module in msgpack (`msgpack._cmsgpack`) does not support PyPy. +But msgpack provides a pure Python implementation (`msgpack.fallback`) for PyPy. ### Windows @@ -82,10 +32,6 @@ Without extension, using pure Python implementation on CPython runs slowly. ## How to use -NOTE: In examples below, I use `raw=False` and `use_bin_type=True` for users -using msgpack < 1.0. These options are default from msgpack 1.0 so you can omit them. - - ### One-shot pack & unpack Use `packb` for packing and `unpackb` for unpacking. @@ -97,16 +43,16 @@ msgpack provides `dumps` and `loads` as an alias for compatibility with ```pycon >>> import msgpack ->>> msgpack.packb([1, 2, 3], use_bin_type=True) +>>> msgpack.packb([1, 2, 3]) '\x93\x01\x02\x03' ->>> msgpack.unpackb(_, raw=False) +>>> msgpack.unpackb(_) [1, 2, 3] ``` `unpack` unpacks msgpack's array to Python's list, but can also unpack to tuple: ```pycon ->>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False) +>>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False) (1, 2, 3) ``` @@ -127,11 +73,11 @@ from io import BytesIO buf = BytesIO() for i in range(100): - buf.write(msgpack.packb(i, use_bin_type=True)) + buf.write(msgpack.packb(i)) buf.seek(0) -unpacker = msgpack.Unpacker(buf, raw=False) +unpacker = msgpack.Unpacker(buf) for unpacked in unpacker: print(unpacked) ``` @@ -162,8 +108,8 @@ def encode_datetime(obj): return obj -packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True) -this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False) +packed_dict = msgpack.packb(useful_dict, default=encode_datetime) +this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime) ``` `Unpacker`'s `object_hook` callback receives a dict; the @@ -191,8 +137,8 @@ It is also possible to pack/unpack custom data types using the **ext** type. ... return ExtType(code, data) ... >>> data = array.array('d', [1.2, 3.4]) ->>> packed = msgpack.packb(data, default=default, use_bin_type=True) ->>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False) +>>> packed = msgpack.packb(data, default=default) +>>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook) >>> data == unpacked True ``` @@ -210,7 +156,7 @@ in a map, can be unpacked or skipped individually. ## Notes -### string and binary type +### string and binary type in old msgpack spec Early versions of msgpack didn't distinguish string and binary types. The type for representing both string and binary types was named **raw**. @@ -263,3 +209,41 @@ You can use `gc.disable()` when unpacking large message. List is the default sequence type of Python. But tuple is lighter than list. You can use `use_list=False` while unpacking when performance is important. + + +## Major breaking changes in the history + +### msgpack 0.5 + +Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5. + +When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before +`pip install -U msgpack`. + + +### msgpack 1.0 + +* Python 2 support + + * The extension module does not support Python 2 anymore. + The pure Python implementation (`msgpack.fallback`) is used for Python 2. + + * msgpack 1.0.6 drops official support of Python 2.7, as pip and + GitHub Action (setup-python) no longer support Python 2.7. + +* Packer + + * Packer uses `use_bin_type=True` by default. + Bytes are encoded in bin type in msgpack. + * The `encoding` option is removed. UTF-8 is used always. + +* Unpacker + + * Unpacker uses `raw=False` by default. It assumes str types are valid UTF-8 string + and decode them to Python str (unicode) object. + * `encoding` option is removed. You can use `raw=True` to support old format (e.g. unpack into bytes, not str). + * Default value of `max_buffer_size` is changed from 0 to 100 MiB to avoid DoS attack. + You need to pass `max_buffer_size=0` if you have large but safe data. + * Default value of `strict_map_key` is changed to True to avoid hashdos. + You need to pass `strict_map_key=False` if you have data which contain map keys + which type is not bytes or str. From 526ec9c923c4867c96537c1f09783fe59432f737 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 16:49:22 +0900 Subject: [PATCH 1142/1172] update cibuildwheel to 2.17 (#601) --- .github/workflows/docs.yaml | 1 - .github/workflows/wheel.yml | 15 ++++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 08fc2f4..b696b92 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -17,7 +17,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.x' - architecture: 'x64' cache: "pip" cache-dependency-path: | requirements.txt diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index e91325b..d57e058 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -14,29 +14,25 @@ jobs: name: Build wheels on ${{ matrix.os }} steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Set up QEMU if: runner.os == 'Linux' uses: docker/setup-qemu-action@v3 with: - platforms: arm64 + platforms: all - - name: Set up Python 3.x - uses: actions/setup-python@v5 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.x" cache: "pip" - - - name: Prepare + - name: Cythonize shell: bash run: | pip install -r requirements.txt make cython - name: Build - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.17.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" @@ -45,6 +41,7 @@ jobs: CIBW_SKIP: pp* - name: Build pure Python wheel + if: runner.os == 'Linux' env: MSGPACK_PUREPYTHON: "1" run: | From 52f8bc2e557ca6684f2f73d129da8317a88bc431 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 5 May 2024 23:14:27 +0900 Subject: [PATCH 1143/1172] implement buffer protocol (#602) Fix #479 --- msgpack/_packer.pyx | 10 +++++++--- msgpack/buff_converter.h | 8 -------- 2 files changed, 7 insertions(+), 11 deletions(-) delete mode 100644 msgpack/buff_converter.h diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 3c39867..c201880 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -44,8 +44,6 @@ cdef extern from "pack.h": int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds); int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit) -cdef extern from "buff_converter.h": - object buff_to_buff(char *, Py_ssize_t) cdef int DEFAULT_RECURSE_LIMIT=511 cdef long long ITEM_LIMIT = (2**32)-1 @@ -371,4 +369,10 @@ cdef class Packer(object): def getbuffer(self): """Return view of internal buffer.""" - return buff_to_buff(self.pk.buf, self.pk.length) + return memoryview(self) + + def __getbuffer__(self, Py_buffer *buffer, int flags): + PyBuffer_FillInfo(buffer, self, self.pk.buf, self.pk.length, 1, flags) + + def __releasebuffer__(self, Py_buffer *buffer): + pass diff --git a/msgpack/buff_converter.h b/msgpack/buff_converter.h deleted file mode 100644 index 86b4196..0000000 --- a/msgpack/buff_converter.h +++ /dev/null @@ -1,8 +0,0 @@ -#include "Python.h" - -/* cython does not support this preprocessor check => write it in raw C */ -static PyObject * -buff_to_buff(char *buff, Py_ssize_t size) -{ - return PyMemoryView_FromMemory(buff, size, PyBUF_READ); -} From a97b31437d3301fa4c29b7813dabee0e690756e8 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 00:13:59 +0900 Subject: [PATCH 1144/1172] Remove unused code (#603) --- msgpack/_packer.pyx | 7 -- msgpack/pack.h | 3 +- msgpack/pack_template.h | 238 ++-------------------------------------- 3 files changed, 8 insertions(+), 240 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index c201880..c7eeda1 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -16,8 +16,6 @@ from .ext import ExtType, Timestamp cdef extern from "Python.h": int PyMemoryView_Check(object obj) - char* PyUnicode_AsUTF8AndSize(object obj, Py_ssize_t *l) except NULL - cdef extern from "pack.h": struct msgpack_packer: @@ -26,11 +24,9 @@ cdef extern from "pack.h": size_t buf_size bint use_bin_type - int msgpack_pack_int(msgpack_packer* pk, int d) int msgpack_pack_nil(msgpack_packer* pk) int msgpack_pack_true(msgpack_packer* pk) int msgpack_pack_false(msgpack_packer* pk) - int msgpack_pack_long(msgpack_packer* pk, long d) int msgpack_pack_long_long(msgpack_packer* pk, long long d) int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) int msgpack_pack_float(msgpack_packer* pk, float d) @@ -184,9 +180,6 @@ cdef class Packer(object): continue else: raise OverflowError("Integer value out of range") - elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o): - longval = o - ret = msgpack_pack_long(&self.pk, longval) elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): if self.use_float: fval = o diff --git a/msgpack/pack.h b/msgpack/pack.h index 901fad7..688eab8 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -21,11 +21,10 @@ #include "sysdep.h" #include #include +#include #ifdef __cplusplus extern "C" { -#else -#define bool char #endif typedef struct msgpack_packer { diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 7d479b6..b8959f0 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -37,18 +37,6 @@ * Integer */ -#define msgpack_pack_real_uint8(x, d) \ -do { \ - if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ - } else { \ - /* unsigned 8 */ \ - unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ -} while(0) - #define msgpack_pack_real_uint16(x, d) \ do { \ if(d < (1<<7)) { \ @@ -123,18 +111,6 @@ do { \ } \ } while(0) -#define msgpack_pack_real_int8(x, d) \ -do { \ - if(d < -(1<<5)) { \ - /* signed 8 */ \ - unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } else { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ - } \ -} while(0) - #define msgpack_pack_real_int16(x, d) \ do { \ if(d < -(1<<5)) { \ @@ -264,49 +240,6 @@ do { \ } while(0) -static inline int msgpack_pack_uint8(msgpack_packer* x, uint8_t d) -{ - msgpack_pack_real_uint8(x, d); -} - -static inline int msgpack_pack_uint16(msgpack_packer* x, uint16_t d) -{ - msgpack_pack_real_uint16(x, d); -} - -static inline int msgpack_pack_uint32(msgpack_packer* x, uint32_t d) -{ - msgpack_pack_real_uint32(x, d); -} - -static inline int msgpack_pack_uint64(msgpack_packer* x, uint64_t d) -{ - msgpack_pack_real_uint64(x, d); -} - -static inline int msgpack_pack_int8(msgpack_packer* x, int8_t d) -{ - msgpack_pack_real_int8(x, d); -} - -static inline int msgpack_pack_int16(msgpack_packer* x, int16_t d) -{ - msgpack_pack_real_int16(x, d); -} - -static inline int msgpack_pack_int32(msgpack_packer* x, int32_t d) -{ - msgpack_pack_real_int32(x, d); -} - -static inline int msgpack_pack_int64(msgpack_packer* x, int64_t d) -{ - msgpack_pack_real_int64(x, d); -} - - -//#ifdef msgpack_pack_inline_func_cint - static inline int msgpack_pack_short(msgpack_packer* x, short d) { #if defined(SIZEOF_SHORT) @@ -372,192 +305,37 @@ if(sizeof(int) == 2) { static inline int msgpack_pack_long(msgpack_packer* x, long d) { #if defined(SIZEOF_LONG) -#if SIZEOF_LONG == 2 - msgpack_pack_real_int16(x, d); -#elif SIZEOF_LONG == 4 +#if SIZEOF_LONG == 4 msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif #elif defined(LONG_MAX) -#if LONG_MAX == 0x7fffL - msgpack_pack_real_int16(x, d); -#elif LONG_MAX == 0x7fffffffL +#if LONG_MAX == 0x7fffffffL msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif #else -if(sizeof(long) == 2) { - msgpack_pack_real_int16(x, d); -} else if(sizeof(long) == 4) { - msgpack_pack_real_int32(x, d); -} else { - msgpack_pack_real_int64(x, d); -} + if (sizeof(long) == 4) { + msgpack_pack_real_int32(x, d); + } else { + msgpack_pack_real_int64(x, d); + } #endif } static inline int msgpack_pack_long_long(msgpack_packer* x, long long d) { -#if defined(SIZEOF_LONG_LONG) -#if SIZEOF_LONG_LONG == 2 - msgpack_pack_real_int16(x, d); -#elif SIZEOF_LONG_LONG == 4 - msgpack_pack_real_int32(x, d); -#else msgpack_pack_real_int64(x, d); -#endif - -#elif defined(LLONG_MAX) -#if LLONG_MAX == 0x7fffL - msgpack_pack_real_int16(x, d); -#elif LLONG_MAX == 0x7fffffffL - msgpack_pack_real_int32(x, d); -#else - msgpack_pack_real_int64(x, d); -#endif - -#else -if(sizeof(long long) == 2) { - msgpack_pack_real_int16(x, d); -} else if(sizeof(long long) == 4) { - msgpack_pack_real_int32(x, d); -} else { - msgpack_pack_real_int64(x, d); -} -#endif -} - -static inline int msgpack_pack_unsigned_short(msgpack_packer* x, unsigned short d) -{ -#if defined(SIZEOF_SHORT) -#if SIZEOF_SHORT == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_SHORT == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(USHRT_MAX) -#if USHRT_MAX == 0xffffU - msgpack_pack_real_uint16(x, d); -#elif USHRT_MAX == 0xffffffffU - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned short) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned short) == 4) { - msgpack_pack_real_uint32(x, d); -} else { - msgpack_pack_real_uint64(x, d); -} -#endif -} - -static inline int msgpack_pack_unsigned_int(msgpack_packer* x, unsigned int d) -{ -#if defined(SIZEOF_INT) -#if SIZEOF_INT == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_INT == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(UINT_MAX) -#if UINT_MAX == 0xffffU - msgpack_pack_real_uint16(x, d); -#elif UINT_MAX == 0xffffffffU - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned int) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned int) == 4) { - msgpack_pack_real_uint32(x, d); -} else { - msgpack_pack_real_uint64(x, d); -} -#endif -} - -static inline int msgpack_pack_unsigned_long(msgpack_packer* x, unsigned long d) -{ -#if defined(SIZEOF_LONG) -#if SIZEOF_LONG == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_LONG == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(ULONG_MAX) -#if ULONG_MAX == 0xffffUL - msgpack_pack_real_uint16(x, d); -#elif ULONG_MAX == 0xffffffffUL - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned long) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned long) == 4) { - msgpack_pack_real_uint32(x, d); -} else { - msgpack_pack_real_uint64(x, d); -} -#endif } static inline int msgpack_pack_unsigned_long_long(msgpack_packer* x, unsigned long long d) { -#if defined(SIZEOF_LONG_LONG) -#if SIZEOF_LONG_LONG == 2 - msgpack_pack_real_uint16(x, d); -#elif SIZEOF_LONG_LONG == 4 - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#elif defined(ULLONG_MAX) -#if ULLONG_MAX == 0xffffUL - msgpack_pack_real_uint16(x, d); -#elif ULLONG_MAX == 0xffffffffUL - msgpack_pack_real_uint32(x, d); -#else - msgpack_pack_real_uint64(x, d); -#endif - -#else -if(sizeof(unsigned long long) == 2) { - msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned long long) == 4) { - msgpack_pack_real_uint32(x, d); -} else { msgpack_pack_real_uint64(x, d); } -#endif -} - -//#undef msgpack_pack_inline_func_cint -//#endif - /* @@ -810,11 +588,9 @@ static inline int msgpack_pack_timestamp(msgpack_packer* x, int64_t seconds, uin #undef TAKE8_32 #undef TAKE8_64 -#undef msgpack_pack_real_uint8 #undef msgpack_pack_real_uint16 #undef msgpack_pack_real_uint32 #undef msgpack_pack_real_uint64 -#undef msgpack_pack_real_int8 #undef msgpack_pack_real_int16 #undef msgpack_pack_real_int32 #undef msgpack_pack_real_int64 From bf2413f915474841aafee2c98321dc465e5f0a3e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 00:30:07 +0900 Subject: [PATCH 1145/1172] ignore msgpack/*.c --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8a06e26..341be63 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ dist/* *.so *~ msgpack/__version__.py +msgpack/*.c msgpack/*.cpp *.egg-info /venv From 72e65feb0e02449fa191346f26b54d3842ab7e69 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 00:49:12 +0900 Subject: [PATCH 1146/1172] packer: add `buf_size` option (#604) And change the default buffer size to 256KiB. Signed-off-by: Rodrigo Tobar Co-authored-by: Rodrigo Tobar --- msgpack/_packer.pyx | 13 +++++++++---- msgpack/_unpacker.pyx | 2 +- msgpack/fallback.py | 27 +++++---------------------- 3 files changed, 15 insertions(+), 27 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index c7eeda1..99557d3 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -53,7 +53,7 @@ cdef inline int PyBytesLike_CheckExact(object o): return PyBytes_CheckExact(o) or PyByteArray_CheckExact(o) -cdef class Packer(object): +cdef class Packer: """ MessagePack Packer @@ -97,6 +97,11 @@ cdef class Packer(object): :param str unicode_errors: The error handler for encoding unicode. (default: 'strict') DO NOT USE THIS!! This option is kept for very specific usage. + + :param int buf_size: + The size of the internal buffer. (default: 256*1024) + Useful if serialisation size can be correctly estimated, + avoid unnecessary reallocations. """ cdef msgpack_packer pk cdef object _default @@ -107,8 +112,7 @@ cdef class Packer(object): cdef bint autoreset cdef bint datetime - def __cinit__(self): - cdef int buf_size = 1024*1024 + def __cinit__(self, buf_size=256*1024, **_kwargs): self.pk.buf = PyMem_Malloc(buf_size) if self.pk.buf == NULL: raise MemoryError("Unable to allocate internal buffer.") @@ -117,7 +121,8 @@ cdef class Packer(object): def __init__(self, *, default=None, bint use_single_float=False, bint autoreset=True, bint use_bin_type=True, - bint strict_types=False, bint datetime=False, unicode_errors=None): + bint strict_types=False, bint datetime=False, unicode_errors=None, + buf_size=256*1024): self.use_float = use_single_float self.strict_types = strict_types self.autoreset = autoreset diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 2771e7b..34ff330 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -210,7 +210,7 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, raise ValueError("Unpack failed: error = %d" % (ret,)) -cdef class Unpacker(object): +cdef class Unpacker: """Streaming unpacker. Arguments: diff --git a/msgpack/fallback.py b/msgpack/fallback.py index ea4c4ce..cbf0d30 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -232,6 +232,7 @@ class Unpacker: def __init__( self, file_like=None, + *, read_size=0, use_list=True, raw=False, @@ -650,32 +651,13 @@ class Packer: The error handler for encoding unicode. (default: 'strict') DO NOT USE THIS!! This option is kept for very specific usage. - Example of streaming deserialize from file-like object:: - - unpacker = Unpacker(file_like) - for o in unpacker: - process(o) - - Example of streaming deserialize from socket:: - - unpacker = Unpacker() - while True: - buf = sock.recv(1024**2) - if not buf: - break - unpacker.feed(buf) - for o in unpacker: - process(o) - - Raises ``ExtraData`` when *packed* contains extra bytes. - Raises ``OutOfData`` when *packed* is incomplete. - Raises ``FormatError`` when *packed* is not valid msgpack. - Raises ``StackError`` when *packed* contains too nested. - Other exceptions can be raised during unpacking. + :param int buf_size: + Internal buffer size. This option is used only for C implementation. """ def __init__( self, + *, default=None, use_single_float=False, autoreset=True, @@ -683,6 +665,7 @@ class Packer: strict_types=False, datetime=False, unicode_errors=None, + buf_size=None, ): self._strict_types = strict_types self._use_float = use_single_float From 3da5818a3a96a4bd39f25a483ee3c22ecb626e9d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 02:12:46 +0900 Subject: [PATCH 1147/1172] update readme (#605) --- README.md | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index d3247bd..61a03c6 100644 --- a/README.md +++ b/README.md @@ -49,17 +49,7 @@ msgpack provides `dumps` and `loads` as an alias for compatibility with [1, 2, 3] ``` -`unpack` unpacks msgpack's array to Python's list, but can also unpack to tuple: - -```pycon ->>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False) -(1, 2, 3) -``` - -You should always specify the `use_list` keyword argument for backward compatibility. -See performance issues relating to `use_list option`_ below. - -Read the docstring for other options. +Read the docstring for options. ### Streaming unpacking @@ -116,6 +106,9 @@ this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime) `object_pairs_hook` callback may instead be used to receive a list of key-value pairs. +NOTE: msgpack can encode datetime with tzinfo into standard ext type for now. +See `datetime` option in `Packer` docstring. + ### Extended types From e1068087e0bb3aebd77c4f2407bf9c429f631787 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 02:13:12 +0900 Subject: [PATCH 1148/1172] cython: better exception handling (#606) - use `except -1` instead of manual error handling - use `PyUnicode_AsUTF8AndSize()` - use `_pack()` and `_pack_inner()` instead of `while True:` --- msgpack/_packer.pyx | 308 +++++++++++++++++++------------------------- msgpack/pack.h | 21 --- 2 files changed, 134 insertions(+), 195 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 99557d3..ad53221 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -24,21 +24,20 @@ cdef extern from "pack.h": size_t buf_size bint use_bin_type - int msgpack_pack_nil(msgpack_packer* pk) - int msgpack_pack_true(msgpack_packer* pk) - int msgpack_pack_false(msgpack_packer* pk) - int msgpack_pack_long_long(msgpack_packer* pk, long long d) - int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) - int msgpack_pack_float(msgpack_packer* pk, float d) - int msgpack_pack_double(msgpack_packer* pk, double d) - int msgpack_pack_array(msgpack_packer* pk, size_t l) - int msgpack_pack_map(msgpack_packer* pk, size_t l) - int msgpack_pack_raw(msgpack_packer* pk, size_t l) - int msgpack_pack_bin(msgpack_packer* pk, size_t l) - int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) - int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l) - int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds); - int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit) + int msgpack_pack_nil(msgpack_packer* pk) except -1 + int msgpack_pack_true(msgpack_packer* pk) except -1 + int msgpack_pack_false(msgpack_packer* pk) except -1 + int msgpack_pack_long_long(msgpack_packer* pk, long long d) except -1 + int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) except -1 + int msgpack_pack_float(msgpack_packer* pk, float d) except -1 + int msgpack_pack_double(msgpack_packer* pk, double d) except -1 + int msgpack_pack_array(msgpack_packer* pk, size_t l) except -1 + int msgpack_pack_map(msgpack_packer* pk, size_t l) except -1 + int msgpack_pack_raw(msgpack_packer* pk, size_t l) except -1 + int msgpack_pack_bin(msgpack_packer* pk, size_t l) except -1 + int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) except -1 + int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l) except -1 + int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds) except -1 cdef int DEFAULT_RECURSE_LIMIT=511 @@ -119,6 +118,10 @@ cdef class Packer: self.pk.buf_size = buf_size self.pk.length = 0 + def __dealloc__(self): + PyMem_Free(self.pk.buf) + self.pk.buf = NULL + def __init__(self, *, default=None, bint use_single_float=False, bint autoreset=True, bint use_bin_type=True, bint strict_types=False, bint datetime=False, unicode_errors=None, @@ -139,155 +142,127 @@ cdef class Packer: else: self.unicode_errors = self._berrors - def __dealloc__(self): - PyMem_Free(self.pk.buf) - self.pk.buf = NULL - - cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: + # returns -2 when default should(o) be called + cdef int _pack_inner(self, object o, bint will_default, int nest_limit) except -1: cdef long long llval cdef unsigned long long ullval cdef unsigned long ulval - cdef long longval - cdef float fval - cdef double dval - cdef char* rawval - cdef int ret - cdef dict d + cdef const char* rawval cdef Py_ssize_t L - cdef int default_used = 0 - cdef bint strict_types = self.strict_types + cdef bool strict_types = self.strict_types cdef Py_buffer view + if o is None: + msgpack_pack_nil(&self.pk) + elif o is True: + msgpack_pack_true(&self.pk) + elif o is False: + msgpack_pack_false(&self.pk) + elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o): + try: + if o > 0: + ullval = o + msgpack_pack_unsigned_long_long(&self.pk, ullval) + else: + llval = o + msgpack_pack_long_long(&self.pk, llval) + except OverflowError as oe: + if will_default: + return -2 + else: + raise OverflowError("Integer value out of range") + elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): + if self.use_float: + msgpack_pack_float(&self.pk, o) + else: + msgpack_pack_double(&self.pk, o) + elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): + L = Py_SIZE(o) + if L > ITEM_LIMIT: + PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) + rawval = o + msgpack_pack_bin(&self.pk, L) + msgpack_pack_raw_body(&self.pk, rawval, L) + elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): + if self.unicode_errors == NULL: + rawval = PyUnicode_AsUTF8AndSize(o, &L) + if L >ITEM_LIMIT: + raise ValueError("unicode string is too large") + else: + o = PyUnicode_AsEncodedString(o, NULL, self.unicode_errors) + L = Py_SIZE(o) + if L > ITEM_LIMIT: + raise ValueError("unicode string is too large") + rawval = o + msgpack_pack_raw(&self.pk, L) + msgpack_pack_raw_body(&self.pk, rawval, L) + elif PyDict_CheckExact(o) if strict_types else PyDict_Check(o): + L = len(o) + if L > ITEM_LIMIT: + raise ValueError("dict is too large") + msgpack_pack_map(&self.pk, L) + for k, v in o.items(): + self._pack(k, nest_limit) + self._pack(v, nest_limit) + elif type(o) is ExtType if strict_types else isinstance(o, ExtType): + # This should be before Tuple because ExtType is namedtuple. + rawval = o.data + L = len(o.data) + if L > ITEM_LIMIT: + raise ValueError("EXT data is too large") + msgpack_pack_ext(&self.pk, o.code, L) + msgpack_pack_raw_body(&self.pk, rawval, L) + elif type(o) is Timestamp: + llval = o.seconds + ulval = o.nanoseconds + msgpack_pack_timestamp(&self.pk, llval, ulval) + elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): + L = Py_SIZE(o) + if L > ITEM_LIMIT: + raise ValueError("list is too large") + msgpack_pack_array(&self.pk, L) + for v in o: + self._pack(v, nest_limit) + elif PyMemoryView_Check(o): + PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) + L = view.len + if L > ITEM_LIMIT: + PyBuffer_Release(&view); + raise ValueError("memoryview is too large") + try: + msgpack_pack_bin(&self.pk, L) + msgpack_pack_raw_body(&self.pk, view.buf, L) + finally: + PyBuffer_Release(&view); + elif self.datetime and PyDateTime_CheckExact(o) and datetime_tzinfo(o) is not None: + delta = o - epoch + if not PyDelta_CheckExact(delta): + raise ValueError("failed to calculate delta") + llval = timedelta_days(delta) * (24*60*60) + timedelta_seconds(delta) + ulval = timedelta_microseconds(delta) * 1000 + msgpack_pack_timestamp(&self.pk, llval, ulval) + elif will_default: + return -2 + elif self.datetime and PyDateTime_CheckExact(o): + # this should be later than will_default + PyErr_Format(ValueError, b"can not serialize '%.200s' object where tzinfo=None", Py_TYPE(o).tp_name) + else: + PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name) + + cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1: + cdef int ret if nest_limit < 0: raise ValueError("recursion limit exceeded.") - - while True: - if o is None: - ret = msgpack_pack_nil(&self.pk) - elif o is True: - ret = msgpack_pack_true(&self.pk) - elif o is False: - ret = msgpack_pack_false(&self.pk) - elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o): - # PyInt_Check(long) is True for Python 3. - # So we should test long before int. - try: - if o > 0: - ullval = o - ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) - else: - llval = o - ret = msgpack_pack_long_long(&self.pk, llval) - except OverflowError as oe: - if not default_used and self._default is not None: - o = self._default(o) - default_used = True - continue - else: - raise OverflowError("Integer value out of range") - elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): - if self.use_float: - fval = o - ret = msgpack_pack_float(&self.pk, fval) - else: - dval = o - ret = msgpack_pack_double(&self.pk, dval) - elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): - L = Py_SIZE(o) - if L > ITEM_LIMIT: - PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) - rawval = o - ret = msgpack_pack_bin(&self.pk, L) - if ret == 0: - ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): - if self.unicode_errors == NULL: - ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT); - if ret == -2: - raise ValueError("unicode string is too large") - else: - o = PyUnicode_AsEncodedString(o, NULL, self.unicode_errors) - L = Py_SIZE(o) - if L > ITEM_LIMIT: - raise ValueError("unicode string is too large") - ret = msgpack_pack_raw(&self.pk, L) - if ret == 0: - rawval = o - ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyDict_CheckExact(o): - d = o - L = len(d) - if L > ITEM_LIMIT: - raise ValueError("dict is too large") - ret = msgpack_pack_map(&self.pk, L) - if ret == 0: - for k, v in d.items(): - ret = self._pack(k, nest_limit-1) - if ret != 0: break - ret = self._pack(v, nest_limit-1) - if ret != 0: break - elif not strict_types and PyDict_Check(o): - L = len(o) - if L > ITEM_LIMIT: - raise ValueError("dict is too large") - ret = msgpack_pack_map(&self.pk, L) - if ret == 0: - for k, v in o.items(): - ret = self._pack(k, nest_limit-1) - if ret != 0: break - ret = self._pack(v, nest_limit-1) - if ret != 0: break - elif type(o) is ExtType if strict_types else isinstance(o, ExtType): - # This should be before Tuple because ExtType is namedtuple. - longval = o.code - rawval = o.data - L = len(o.data) - if L > ITEM_LIMIT: - raise ValueError("EXT data is too large") - ret = msgpack_pack_ext(&self.pk, longval, L) - ret = msgpack_pack_raw_body(&self.pk, rawval, L) - elif type(o) is Timestamp: - llval = o.seconds - ulval = o.nanoseconds - ret = msgpack_pack_timestamp(&self.pk, llval, ulval) - elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): - L = Py_SIZE(o) - if L > ITEM_LIMIT: - raise ValueError("list is too large") - ret = msgpack_pack_array(&self.pk, L) - if ret == 0: - for v in o: - ret = self._pack(v, nest_limit-1) - if ret != 0: break - elif PyMemoryView_Check(o): - if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0: - raise ValueError("could not get buffer for memoryview") - L = view.len - if L > ITEM_LIMIT: - PyBuffer_Release(&view); - raise ValueError("memoryview is too large") - ret = msgpack_pack_bin(&self.pk, L) - if ret == 0: - ret = msgpack_pack_raw_body(&self.pk, view.buf, L) - PyBuffer_Release(&view); - elif self.datetime and PyDateTime_CheckExact(o) and datetime_tzinfo(o) is not None: - delta = o - epoch - if not PyDelta_CheckExact(delta): - raise ValueError("failed to calculate delta") - llval = timedelta_days(delta) * (24*60*60) + timedelta_seconds(delta) - ulval = timedelta_microseconds(delta) * 1000 - ret = msgpack_pack_timestamp(&self.pk, llval, ulval) - elif not default_used and self._default: + nest_limit -= 1 + if self._default is not None: + ret = self._pack_inner(o, 1, nest_limit) + if ret == -2: o = self._default(o) - default_used = 1 - continue - elif self.datetime and PyDateTime_CheckExact(o): - PyErr_Format(ValueError, b"can not serialize '%.200s' object where tzinfo=None", Py_TYPE(o).tp_name) else: - PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name) - return ret + return ret + return self._pack_inner(o, 0, nest_limit) - cpdef pack(self, object obj): + def pack(self, object obj): cdef int ret try: ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) @@ -308,11 +283,7 @@ cdef class Packer: def pack_array_header(self, long long size): if size > ITEM_LIMIT: raise ValueError - cdef int ret = msgpack_pack_array(&self.pk, size) - if ret == -1: - raise MemoryError - elif ret: # should not happen - raise TypeError + msgpack_pack_array(&self.pk, size) if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 @@ -321,11 +292,7 @@ cdef class Packer: def pack_map_header(self, long long size): if size > ITEM_LIMIT: raise ValueError - cdef int ret = msgpack_pack_map(&self.pk, size) - if ret == -1: - raise MemoryError - elif ret: # should not happen - raise TypeError + msgpack_pack_map(&self.pk, size) if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 @@ -338,17 +305,10 @@ cdef class Packer: *pairs* should be a sequence of pairs. (`len(pairs)` and `for k, v in pairs:` should be supported.) """ - cdef int ret = msgpack_pack_map(&self.pk, len(pairs)) - if ret == 0: - for k, v in pairs: - ret = self._pack(k) - if ret != 0: break - ret = self._pack(v) - if ret != 0: break - if ret == -1: - raise MemoryError - elif ret: # should not happen - raise TypeError + msgpack_pack_map(&self.pk, len(pairs)) + for k, v in pairs: + self._pack(k) + self._pack(v) if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 diff --git a/msgpack/pack.h b/msgpack/pack.h index 688eab8..edf3a3f 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -64,27 +64,6 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ #include "pack_template.h" -// return -2 when o is too long -static inline int -msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit) -{ - assert(PyUnicode_Check(o)); - - Py_ssize_t len; - const char* buf = PyUnicode_AsUTF8AndSize(o, &len); - if (buf == NULL) - return -1; - - if (len > limit) { - return -2; - } - - int ret = msgpack_pack_raw(pk, len); - if (ret) return ret; - - return msgpack_pack_raw_body(pk, buf, len); -} - #ifdef __cplusplus } #endif From e0f0e145f15364819bac80bd0808834a9df0065e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 03:33:48 +0900 Subject: [PATCH 1149/1172] better error checks (#607) * check buffer exports * add error messages --- msgpack/_packer.pyx | 48 +++++++++++++++++++++++++++++++++------------ test/test_buffer.py | 24 +++++++++++++++++++++-- 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index ad53221..402b694 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -106,6 +106,7 @@ cdef class Packer: cdef object _default cdef object _berrors cdef const char *unicode_errors + cdef size_t exports # number of exported buffers cdef bint strict_types cdef bint use_float cdef bint autoreset @@ -117,10 +118,16 @@ cdef class Packer: raise MemoryError("Unable to allocate internal buffer.") self.pk.buf_size = buf_size self.pk.length = 0 + self.exports = 0 def __dealloc__(self): PyMem_Free(self.pk.buf) self.pk.buf = NULL + assert self.exports == 0 + + cdef _check_exports(self): + if self.exports > 0: + raise BufferError("Existing exports of data: Packer cannot be changed") def __init__(self, *, default=None, bint use_single_float=False, bint autoreset=True, bint use_bin_type=True, @@ -149,8 +156,8 @@ cdef class Packer: cdef unsigned long ulval cdef const char* rawval cdef Py_ssize_t L - cdef bool strict_types = self.strict_types cdef Py_buffer view + cdef bint strict = self.strict_types if o is None: msgpack_pack_nil(&self.pk) @@ -158,7 +165,7 @@ cdef class Packer: msgpack_pack_true(&self.pk) elif o is False: msgpack_pack_false(&self.pk) - elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o): + elif PyLong_CheckExact(o) if strict else PyLong_Check(o): try: if o > 0: ullval = o @@ -171,19 +178,19 @@ cdef class Packer: return -2 else: raise OverflowError("Integer value out of range") - elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o): + elif PyFloat_CheckExact(o) if strict else PyFloat_Check(o): if self.use_float: msgpack_pack_float(&self.pk, o) else: msgpack_pack_double(&self.pk, o) - elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o): + elif PyBytesLike_CheckExact(o) if strict else PyBytesLike_Check(o): L = Py_SIZE(o) if L > ITEM_LIMIT: PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name) rawval = o msgpack_pack_bin(&self.pk, L) msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o): + elif PyUnicode_CheckExact(o) if strict else PyUnicode_Check(o): if self.unicode_errors == NULL: rawval = PyUnicode_AsUTF8AndSize(o, &L) if L >ITEM_LIMIT: @@ -196,7 +203,7 @@ cdef class Packer: rawval = o msgpack_pack_raw(&self.pk, L) msgpack_pack_raw_body(&self.pk, rawval, L) - elif PyDict_CheckExact(o) if strict_types else PyDict_Check(o): + elif PyDict_CheckExact(o) if strict else PyDict_Check(o): L = len(o) if L > ITEM_LIMIT: raise ValueError("dict is too large") @@ -204,7 +211,7 @@ cdef class Packer: for k, v in o.items(): self._pack(k, nest_limit) self._pack(v, nest_limit) - elif type(o) is ExtType if strict_types else isinstance(o, ExtType): + elif type(o) is ExtType if strict else isinstance(o, ExtType): # This should be before Tuple because ExtType is namedtuple. rawval = o.data L = len(o.data) @@ -216,7 +223,7 @@ cdef class Packer: llval = o.seconds ulval = o.nanoseconds msgpack_pack_timestamp(&self.pk, llval, ulval) - elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)): + elif PyList_CheckExact(o) if strict else (PyTuple_Check(o) or PyList_Check(o)): L = Py_SIZE(o) if L > ITEM_LIMIT: raise ValueError("list is too large") @@ -264,6 +271,7 @@ cdef class Packer: def pack(self, object obj): cdef int ret + self._check_exports() try: ret = self._pack(obj, DEFAULT_RECURSE_LIMIT) except: @@ -277,12 +285,16 @@ cdef class Packer: return buf def pack_ext_type(self, typecode, data): + self._check_exports() + if len(data) > ITEM_LIMIT: + raise ValueError("ext data too large") msgpack_pack_ext(&self.pk, typecode, len(data)) msgpack_pack_raw_body(&self.pk, data, len(data)) def pack_array_header(self, long long size): + self._check_exports() if size > ITEM_LIMIT: - raise ValueError + raise ValueError("array too large") msgpack_pack_array(&self.pk, size) if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) @@ -290,8 +302,9 @@ cdef class Packer: return buf def pack_map_header(self, long long size): + self._check_exports() if size > ITEM_LIMIT: - raise ValueError + raise ValueError("map too learge") msgpack_pack_map(&self.pk, size) if self.autoreset: buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) @@ -305,7 +318,11 @@ cdef class Packer: *pairs* should be a sequence of pairs. (`len(pairs)` and `for k, v in pairs:` should be supported.) """ - msgpack_pack_map(&self.pk, len(pairs)) + self._check_exports() + size = len(pairs) + if size > ITEM_LIMIT: + raise ValueError("map too large") + msgpack_pack_map(&self.pk, size) for k, v in pairs: self._pack(k) self._pack(v) @@ -319,6 +336,7 @@ cdef class Packer: This method is useful only when autoreset=False. """ + self._check_exports() self.pk.length = 0 def bytes(self): @@ -326,11 +344,15 @@ cdef class Packer: return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) def getbuffer(self): - """Return view of internal buffer.""" + """Return memoryview of internal buffer. + + Note: Packer now supports buffer protocol. You can use memoryview(packer). + """ return memoryview(self) def __getbuffer__(self, Py_buffer *buffer, int flags): PyBuffer_FillInfo(buffer, self, self.pk.buf, self.pk.length, 1, flags) + self.exports += 1 def __releasebuffer__(self, Py_buffer *buffer): - pass + self.exports -= 1 diff --git a/test/test_buffer.py b/test/test_buffer.py index a3db339..2165eb5 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +from pytest import raises -from msgpack import packb, unpackb +from msgpack import packb, unpackb, Packer def test_unpack_buffer(): @@ -27,3 +27,23 @@ def test_unpack_memoryview(): assert [b"foo", b"bar"] == obj expected_type = bytes assert all(type(s) == expected_type for s in obj) + + +def test_packer_getbuffer(): + packer = Packer(autoreset=False) + packer.pack_array_header(2) + packer.pack(42) + packer.pack("hello") + buffer = packer.getbuffer() + assert isinstance(buffer, memoryview) + assert bytes(buffer) == b"\x92*\xa5hello" + + if Packer.__module__ == "msgpack._cmsgpack": # only for Cython + # cython Packer supports buffer protocol directly + assert bytes(packer) == b"\x92*\xa5hello" + + with raises(BufferError): + packer.pack(42) + buffer.release() + packer.pack(42) + assert bytes(packer) == b"\x92*\xa5hello*" From 33e0e86f4e66e6bb029f118e8f5e166b55e828f7 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 May 2024 11:46:31 +0900 Subject: [PATCH 1150/1172] Cleanup code and pyproject (#608) * use isort * fallback: use BytesIO instead of StringIO. We had dropped Python 2 already. --- Makefile | 4 +-- msgpack/__init__.py | 12 ++++----- msgpack/ext.py | 2 +- msgpack/fallback.py | 54 ++++++++++++++++++----------------------- pyproject.toml | 17 +++++-------- setup.py | 4 +-- test/test_buffer.py | 2 +- test/test_except.py | 7 +++--- test/test_extension.py | 1 + test/test_limits.py | 8 +++--- test/test_memoryview.py | 1 + test/test_newspec.py | 2 +- test/test_obj.py | 1 + test/test_pack.py | 4 +-- test/test_read_size.py | 2 +- test/test_seq.py | 6 ++--- test/test_sequnpack.py | 7 +++--- test/test_stricttype.py | 3 ++- test/test_subtype.py | 3 ++- test/test_timestamp.py | 4 ++- test/test_unpack.py | 11 +++------ 21 files changed, 75 insertions(+), 80 deletions(-) diff --git a/Makefile b/Makefile index 3ce178f..51f3e0e 100644 --- a/Makefile +++ b/Makefile @@ -6,11 +6,11 @@ all: cython .PHONY: format format: - pipx run ruff format $(PYTHON_SOURCES) + ruff format $(PYTHON_SOURCES) .PHONY: lint lint: - pipx run ruff check $(PYTHON_SOURCES) + ruff check $(PYTHON_SOURCES) .PHONY: doc doc: diff --git a/msgpack/__init__.py b/msgpack/__init__.py index 919b86f..e796efb 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -1,20 +1,20 @@ -from .exceptions import * -from .ext import ExtType, Timestamp - +# ruff: noqa: F401 import os +from .exceptions import * # noqa: F403 +from .ext import ExtType, Timestamp version = (1, 0, 8) __version__ = "1.0.8" if os.environ.get("MSGPACK_PUREPYTHON"): - from .fallback import Packer, unpackb, Unpacker + from .fallback import Packer, Unpacker, unpackb else: try: - from ._cmsgpack import Packer, unpackb, Unpacker + from ._cmsgpack import Packer, Unpacker, unpackb except ImportError: - from .fallback import Packer, unpackb, Unpacker + from .fallback import Packer, Unpacker, unpackb def pack(o, stream, **kwargs): diff --git a/msgpack/ext.py b/msgpack/ext.py index 3940fe0..9694819 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -1,6 +1,6 @@ -from collections import namedtuple import datetime import struct +from collections import namedtuple class ExtType(namedtuple("ExtType", "code data")): diff --git a/msgpack/fallback.py b/msgpack/fallback.py index cbf0d30..b02e47c 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -1,28 +1,22 @@ """Fallback pure Python implementation of msgpack""" -from datetime import datetime as _DateTime -import sys import struct - +import sys +from datetime import datetime as _DateTime if hasattr(sys, "pypy_version_info"): - # StringIO is slow on PyPy, StringIO is faster. However: PyPy's own - # StringBuilder is fastest. from __pypy__ import newlist_hint + from __pypy__.builders import BytesBuilder - try: - from __pypy__.builders import BytesBuilder as StringBuilder - except ImportError: - from __pypy__.builders import StringBuilder - USING_STRINGBUILDER = True + _USING_STRINGBUILDER = True - class StringIO: + class BytesIO: def __init__(self, s=b""): if s: - self.builder = StringBuilder(len(s)) + self.builder = BytesBuilder(len(s)) self.builder.append(s) else: - self.builder = StringBuilder() + self.builder = BytesBuilder() def write(self, s): if isinstance(s, memoryview): @@ -35,17 +29,17 @@ if hasattr(sys, "pypy_version_info"): return self.builder.build() else: - USING_STRINGBUILDER = False - from io import BytesIO as StringIO + from io import BytesIO - newlist_hint = lambda size: [] + _USING_STRINGBUILDER = False + + def newlist_hint(size): + return [] -from .exceptions import BufferFull, OutOfData, ExtraData, FormatError, StackError - +from .exceptions import BufferFull, ExtraData, FormatError, OutOfData, StackError from .ext import ExtType, Timestamp - EX_SKIP = 0 EX_CONSTRUCT = 1 EX_READ_ARRAY_HEADER = 2 @@ -335,6 +329,7 @@ class Unpacker: # Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython self._buffer.extend(view) + view.release() def _consume(self): """Gets rid of the used parts of the buffer.""" @@ -671,12 +666,11 @@ class Packer: self._use_float = use_single_float self._autoreset = autoreset self._use_bin_type = use_bin_type - self._buffer = StringIO() + self._buffer = BytesIO() self._datetime = bool(datetime) self._unicode_errors = unicode_errors or "strict" - if default is not None: - if not callable(default): - raise TypeError("default must be callable") + if default is not None and not callable(default): + raise TypeError("default must be callable") self._default = default def _pack( @@ -807,18 +801,18 @@ class Packer: try: self._pack(obj) except: - self._buffer = StringIO() # force reset + self._buffer = BytesIO() # force reset raise if self._autoreset: ret = self._buffer.getvalue() - self._buffer = StringIO() + self._buffer = BytesIO() return ret def pack_map_pairs(self, pairs): self._pack_map_pairs(len(pairs), pairs) if self._autoreset: ret = self._buffer.getvalue() - self._buffer = StringIO() + self._buffer = BytesIO() return ret def pack_array_header(self, n): @@ -827,7 +821,7 @@ class Packer: self._pack_array_header(n) if self._autoreset: ret = self._buffer.getvalue() - self._buffer = StringIO() + self._buffer = BytesIO() return ret def pack_map_header(self, n): @@ -836,7 +830,7 @@ class Packer: self._pack_map_header(n) if self._autoreset: ret = self._buffer.getvalue() - self._buffer = StringIO() + self._buffer = BytesIO() return ret def pack_ext_type(self, typecode, data): @@ -925,11 +919,11 @@ class Packer: This method is useful only when autoreset=False. """ - self._buffer = StringIO() + self._buffer = BytesIO() def getbuffer(self): """Return view of internal buffer.""" - if USING_STRINGBUILDER: + if _USING_STRINGBUILDER: return memoryview(self.bytes()) else: return self._buffer.getbuffer() diff --git a/pyproject.toml b/pyproject.toml index 6254f06..f36c7f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,17 +45,12 @@ include-package-data = false [tool.setuptools.dynamic] version = {attr = "msgpack.__version__"} -[tool.black] -line-length = 100 -target-version = ["py37"] -skip_string_normalization = true - [tool.ruff] line-length = 100 target-version = "py38" -lint.ignore = [] - -[tool.ruff.lint.per-file-ignores] -"msgpack/__init__.py" = ["F401", "F403"] -"msgpack/fallback.py" = ["E731"] -"test/test_seq.py" = ["E501"] +lint.select = [ + "E", # pycodestyle + "F", # Pyflakes + "I", # isort + #"UP", pyupgrade +] diff --git a/setup.py b/setup.py index dc14a26..eaca746 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,11 @@ #!/usr/bin/env python import os import sys -from setuptools import setup, Extension + +from setuptools import Extension, setup from setuptools.command.build_ext import build_ext from setuptools.command.sdist import sdist - PYPY = hasattr(sys, "pypy_version_info") diff --git a/test/test_buffer.py b/test/test_buffer.py index 2165eb5..2c5a14c 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -1,6 +1,6 @@ from pytest import raises -from msgpack import packb, unpackb, Packer +from msgpack import Packer, packb, unpackb def test_unpack_buffer(): diff --git a/test/test_except.py b/test/test_except.py index 8c0a976..b77ac80 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -1,10 +1,11 @@ #!/usr/bin/env python -from pytest import raises -from msgpack import packb, unpackb, Unpacker, FormatError, StackError, OutOfData - import datetime +from pytest import raises + +from msgpack import FormatError, OutOfData, StackError, Unpacker, packb, unpackb + class DummyException(Exception): pass diff --git a/test/test_extension.py b/test/test_extension.py index 9e5e6aa..aaf0fd9 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,4 +1,5 @@ import array + import msgpack from msgpack import ExtType diff --git a/test/test_limits.py b/test/test_limits.py index 533bc11..9b92b4d 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -2,14 +2,14 @@ import pytest from msgpack import ( - packb, - unpackb, - Packer, - Unpacker, ExtType, + Packer, PackOverflowError, PackValueError, + Unpacker, UnpackValueError, + packb, + unpackb, ) diff --git a/test/test_memoryview.py b/test/test_memoryview.py index eff4bca..0a2a6f5 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from array import array + from msgpack import packb, unpackb diff --git a/test/test_newspec.py b/test/test_newspec.py index a6f4251..9e2f9be 100644 --- a/test/test_newspec.py +++ b/test/test_newspec.py @@ -1,4 +1,4 @@ -from msgpack import packb, unpackb, ExtType +from msgpack import ExtType, packb, unpackb def test_str8(): diff --git a/test/test_obj.py b/test/test_obj.py index f78bf42..23be06d 100644 --- a/test/test_obj.py +++ b/test/test_obj.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from pytest import raises + from msgpack import packb, unpackb diff --git a/test/test_pack.py b/test/test_pack.py index 4a0ef40..374d154 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -1,12 +1,12 @@ #!/usr/bin/env python +import struct from collections import OrderedDict from io import BytesIO -import struct import pytest -from msgpack import packb, unpackb, Unpacker, Packer +from msgpack import Packer, Unpacker, packb, unpackb def check(data, use_list=False): diff --git a/test/test_read_size.py b/test/test_read_size.py index a7d61fd..0f6c1b5 100644 --- a/test/test_read_size.py +++ b/test/test_read_size.py @@ -1,6 +1,6 @@ """Test Unpacker's read_array_header and read_map_header methods""" -from msgpack import packb, Unpacker, OutOfData +from msgpack import OutOfData, Unpacker, packb UnexpectedTypeException = ValueError diff --git a/test/test_seq.py b/test/test_seq.py index 16d9dde..8dee462 100644 --- a/test/test_seq.py +++ b/test/test_seq.py @@ -1,8 +1,8 @@ -#!/usr/bin/env python - +# ruff: noqa: E501 +# ignore line length limit for long comments import io -import msgpack +import msgpack binarydata = bytes(bytearray(range(256))) diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 6b138aa..0f895d7 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -1,10 +1,11 @@ #!/usr/bin/env python import io -from msgpack import Unpacker, BufferFull -from msgpack import pack, packb -from msgpack.exceptions import OutOfData + from pytest import raises +from msgpack import BufferFull, Unpacker, pack, packb +from msgpack.exceptions import OutOfData + def test_partialdata(): unpacker = Unpacker() diff --git a/test/test_stricttype.py b/test/test_stricttype.py index 9ffaff2..72776a2 100644 --- a/test/test_stricttype.py +++ b/test/test_stricttype.py @@ -1,5 +1,6 @@ from collections import namedtuple -from msgpack import packb, unpackb, ExtType + +from msgpack import ExtType, packb, unpackb def test_namedtuple(): diff --git a/test/test_subtype.py b/test/test_subtype.py index 0d1c41a..a911578 100644 --- a/test/test_subtype.py +++ b/test/test_subtype.py @@ -1,8 +1,9 @@ #!/usr/bin/env python -from msgpack import packb from collections import namedtuple +from msgpack import packb + class MyList(list): pass diff --git a/test/test_timestamp.py b/test/test_timestamp.py index f9bc835..831141a 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -1,5 +1,7 @@ -import pytest import datetime + +import pytest + import msgpack from msgpack.ext import Timestamp diff --git a/test/test_unpack.py b/test/test_unpack.py index bf3f960..b17c3c5 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -1,12 +1,9 @@ -from io import BytesIO import sys -from msgpack import Unpacker, packb, OutOfData, ExtType -from pytest import raises, mark +from io import BytesIO -try: - from itertools import izip as zip -except ImportError: - pass +from pytest import mark, raises + +from msgpack import ExtType, OutOfData, Unpacker, packb def test_unpack_array_header_from_file(): From 9cea8b6da23ce66f0e78f017c96adcc447deb09a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 7 May 2024 20:49:23 +0900 Subject: [PATCH 1151/1172] Release v1.1.0rc1 (#609) --- ChangeLog.rst | 20 +++++++++++++++++++- msgpack/__init__.py | 4 ++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 2408bc9..9b16e41 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,21 @@ +1.1.0rc1 +======== + +Release Date: 2024-05-07 + +* Update Cython to 3.0.10 to reduce C warnings and future support for Python 3.13. +* Stop using C++ mode in Cython to reduce compile error on some compilers. +* ``Packer()`` has ``buf_size`` option to specify initial size of + internal buffer to reduce reallocation. +* The default internal buffer size of ``Packer()`` is reduced from + 1MiB to 256KiB to optimize for common use cases. Use ``buf_size`` + if you are packing large data. +* ``Timestamp.to_datetime()`` and ``Timestamp.from_datetime()`` become + more accurate by avoiding floating point calculations. (#591) +* The Cython code for ``Unpacker`` has been slightly rewritten for maintainability. +* The fallback implementation of ``Packer()`` and ``Unpacker()`` now uses keyword-only + arguments to improve compatibility with the Cython implementation. + 1.0.8 ===== @@ -130,7 +148,7 @@ Important changes * unpacker: Default value of input limits are smaller than before to avoid DoS attack. If you need to handle large data, you need to specify limits manually. (#319) -* Unpacker doesn't wrap underlaying ``ValueError`` (including ``UnicodeError``) into +* Unpacker doesn't wrap underlying ``ValueError`` (including ``UnicodeError``) into ``UnpackValueError``. If you want to catch all exception during unpack, you need to use ``try ... except Exception`` with minimum try code block. (#323, #233) diff --git a/msgpack/__init__.py b/msgpack/__init__.py index e796efb..c252544 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ import os from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 0, 8) -__version__ = "1.0.8" +version = (1, 1, 0, "rc1") +__version__ = "1.1.0rc1" if os.environ.get("MSGPACK_PUREPYTHON"): From 0b1c47b06b55d91c00c9f7153c4a9440ea878886 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 7 May 2024 22:01:54 +0900 Subject: [PATCH 1152/1172] do not install cython as build dependency (#610) User can not cythonize during `pip install msgpack`. So remove cython from build dependency. If user need to use another Cython, user should download sdist, unzip, manually cythonize, and `pip install .`. --- pyproject.toml | 6 +----- setup.py | 45 --------------------------------------------- 2 files changed, 1 insertion(+), 50 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f36c7f4..d041d4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,5 @@ [build-system] -requires = [ - # Also declared in requirements.txt, if updating here please also update there - "Cython~=3.0.10", - "setuptools >= 69.5.1", -] +requires = ["setuptools >= 69.5.1"] build-backend = "setuptools.build_meta" [project] diff --git a/setup.py b/setup.py index eaca746..4029e9e 100644 --- a/setup.py +++ b/setup.py @@ -3,52 +3,9 @@ import os import sys from setuptools import Extension, setup -from setuptools.command.build_ext import build_ext -from setuptools.command.sdist import sdist PYPY = hasattr(sys, "pypy_version_info") - -class NoCython(Exception): - pass - - -try: - import Cython.Compiler.Main as cython_compiler - - have_cython = True -except ImportError: - have_cython = False - - -def cythonize(src): - if not have_cython: - raise Exception("Cython is required for building from checkout") - sys.stderr.write(f"cythonize: {src!r}\n") - cython_compiler.compile([src]) - - -def ensure_source(src): - pyx = os.path.splitext(src)[0] + ".pyx" - - if not os.path.exists(src) or have_cython and os.stat(src).st_mtime < os.stat(pyx).st_mtime: - cythonize(pyx) - - -class BuildExt(build_ext): - def build_extension(self, ext): - for src in ext.sources: - ensure_source(src) - return build_ext.build_extension(self, ext) - - -# Cython is required for sdist -class Sdist(sdist): - def __init__(self, *args, **kwargs): - cythonize("msgpack/_cmsgpack.pyx") - sdist.__init__(self, *args, **kwargs) - - libraries = [] macros = [] ext_modules = [] @@ -69,9 +26,7 @@ if not PYPY and not os.environ.get("MSGPACK_PUREPYTHON"): ) del libraries, macros - setup( - cmdclass={"build_ext": BuildExt, "sdist": Sdist}, ext_modules=ext_modules, packages=["msgpack"], ) From 6e11368f5d54f7d4878dc209717495c37be03c68 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 19 Aug 2024 17:35:16 +0900 Subject: [PATCH 1153/1172] update Cython to 3.0.11 (#617) --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1164a94..b677f06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -# Also declared in pyproject.toml, if updating here please also update there. -Cython~=3.0.10 +Cython~=3.0.11 From 9e26d80ab2a02221d3ca36cc0b5ca2268f391204 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 19 Aug 2024 17:56:01 +0900 Subject: [PATCH 1154/1172] update cibuildwheel to 2.20.0 (#618) --- .github/workflows/wheel.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index d57e058..01d0bbd 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -32,7 +32,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.17.0 + uses: pypa/cibuildwheel@v2.20.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" @@ -40,13 +40,11 @@ jobs: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_SKIP: pp* - - name: Build pure Python wheel + - name: Build sdist if: runner.os == 'Linux' - env: - MSGPACK_PUREPYTHON: "1" run: | pip install build - python -m build -w -o wheelhouse + python -m build -s -o wheelhouse - name: Upload Wheels to artifact uses: actions/upload-artifact@v4 From 9d0c7f2f9cab21c3e39d23001cd9d2034824ee61 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 19 Aug 2024 20:36:26 +0900 Subject: [PATCH 1155/1172] Release v1.1.0rc2 (#619) --- .github/workflows/wheel.yml | 2 +- ChangeLog.rst | 8 ++++++++ msgpack/__init__.py | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 01d0bbd..5015722 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -38,7 +38,7 @@ jobs: CIBW_TEST_COMMAND: "pytest {package}/test" CIBW_ARCHS_LINUX: auto aarch64 CIBW_ARCHS_MACOS: x86_64 universal2 arm64 - CIBW_SKIP: pp* + CIBW_SKIP: "pp* cp38-macosx_*" - name: Build sdist if: runner.os == 'Linux' diff --git a/ChangeLog.rst b/ChangeLog.rst index 9b16e41..47328ff 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,11 @@ +1.1.0rc2 +======== + +Release Date: 2024-08-19 + +* Update Cython to 3.0.11 for better Python 3.13 support. +* Update cibuildwheel to 2.20.0 to build Python 3.13 wheels. + 1.1.0rc1 ======== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index c252544..a72e974 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ import os from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 0, "rc1") -__version__ = "1.1.0rc1" +version = (1, 1, 0, "rc2") +__version__ = "1.1.0rc2" if os.environ.get("MSGPACK_PUREPYTHON"): From 20a2b8eaa26d76169049ce150e10d1d2aa37d3ab Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 21 Aug 2024 01:56:00 -0400 Subject: [PATCH 1156/1172] use `PyLong_*` instead of `PyInt_*` (#620) 9af421163cb8081414be347038dee7a82b29e8dd in Cython removed back-compatibility `#define`. --- msgpack/unpack.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 23aa622..58a2f4f 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -47,7 +47,7 @@ static inline msgpack_unpack_object unpack_callback_root(unpack_user* u) static inline int unpack_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) { - PyObject *p = PyInt_FromLong((long)d); + PyObject *p = PyLong_FromLong((long)d); if (!p) return -1; *o = p; @@ -61,7 +61,7 @@ static inline int unpack_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpac static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { - PyObject *p = PyInt_FromSize_t((size_t)d); + PyObject *p = PyLong_FromSize_t((size_t)d); if (!p) return -1; *o = p; @@ -74,7 +74,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp if (d > LONG_MAX) { p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d); } else { - p = PyInt_FromLong((long)d); + p = PyLong_FromLong((long)d); } if (!p) return -1; @@ -84,7 +84,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp static inline int unpack_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) { - PyObject *p = PyInt_FromLong(d); + PyObject *p = PyLong_FromLong(d); if (!p) return -1; *o = p; @@ -107,7 +107,7 @@ static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpac if (d > LONG_MAX || d < LONG_MIN) { p = PyLong_FromLongLong((PY_LONG_LONG)d); } else { - p = PyInt_FromLong((long)d); + p = PyLong_FromLong((long)d); } *o = p; return 0; From 4587393b1ae2c9ebbd5bc93005b8aea2c8050b27 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 10 Sep 2024 01:58:00 +0900 Subject: [PATCH 1157/1172] release v1.1.0 (#622) --- ChangeLog.rst | 8 ++++++++ msgpack/__init__.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 47328ff..863c6b2 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,11 @@ +1.1.0 +===== + +Release Date: 2024-09-10 + +* use ``PyLong_*`` instead of ``PyInt_*`` for compatibility with + future Cython. (#620) + 1.1.0rc2 ======== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index a72e974..b615105 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ import os from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 0, "rc2") -__version__ = "1.1.0rc2" +version = (1, 1, 0) +__version__ = "1.1.0" if os.environ.get("MSGPACK_PUREPYTHON"): From 0eeabfb453844b441a4a77097b3d5aa0cb6645b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Tue, 8 Oct 2024 03:04:56 -0600 Subject: [PATCH 1158/1172] Add Python 3.13 trove classifier (#626) --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index d041d4c..e24f2b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Intended Audience :: Developers", From 868aa2cd83f39237deb957c68ce7232422a5950b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 31 May 2025 12:45:06 +0900 Subject: [PATCH 1159/1172] update Cython to 3.1.1 (#637) --- .github/workflows/test.yml | 9 +++++++-- pyproject.toml | 12 +++++++----- requirements.txt | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 530238c..23d221c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.13-dev", "3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9", "3.8"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -26,10 +26,15 @@ jobs: allow-prereleases: true cache: "pip" + - name: Prepare + shell: bash + run: | + pip install -U pip + pip install -r requirements.txt pytest + - name: Build shell: bash run: | - pip install -r requirements.txt pytest make cython pip install . diff --git a/pyproject.toml b/pyproject.toml index e24f2b8..b162832 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,13 @@ [build-system] -requires = ["setuptools >= 69.5.1"] +# 75.3.0 is the latest version supporting Python 3.8 +requires = ["setuptools >= 75.3.0"] build-backend = "setuptools.build_meta" [project] name = "msgpack" dynamic = ["version"] +# `license = "Apache-2.0"` is preferred. But keep old syntax for Python 3.8 compatibility. +# https://github.com/msgpack/msgpack-python/pull/637 license = {text="Apache 2.0"} authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}] description = "MessagePack serializer" @@ -14,18 +17,17 @@ requires-python = ">=3.8" classifiers = [ "Development Status :: 5 - Production/Stable", "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", + "Topic :: File Formats", + "Intended Audience :: Developers", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", ] [project.urls] diff --git a/requirements.txt b/requirements.txt index b677f06..78a2f38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython~=3.0.11 +Cython~=3.1.1 From cdc764450370ff80e7c83edbe8d015f08f6fb9b3 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 1 Jun 2025 16:56:44 +0900 Subject: [PATCH 1160/1172] update cibuildwheel to v2.23.3 (#638) --- .github/workflows/wheel.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 5015722..d97de1d 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -9,17 +9,12 @@ jobs: build_wheels: strategy: matrix: - os: ["ubuntu-latest", "windows-latest", "macos-latest"] + # macos-13 is for intel + os: ["ubuntu-24.04", "ubuntu-24.04-arm", "windows-latest", "macos-13", "macos-latest"] runs-on: ${{ matrix.os }} name: Build wheels on ${{ matrix.os }} steps: - - name: Set up QEMU - if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v3 - with: - platforms: all - - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: @@ -32,12 +27,10 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.20.0 + uses: pypa/cibuildwheel@v2.23.3 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" - CIBW_ARCHS_LINUX: auto aarch64 - CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_SKIP: "pp* cp38-macosx_*" - name: Build sdist From fe9e620a607702b31476f092ad01a387cff4cfbd Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 2 Jun 2025 14:46:53 +0900 Subject: [PATCH 1161/1172] upload to PyPI on create a release (#639) --- .github/workflows/wheel.yml | 48 ++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index d97de1d..686d7dd 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -1,8 +1,10 @@ -name: Build Wheels +name: Build sdist and Wheels on: push: branches: [main] - create: + release: + types: + - published workflow_dispatch: jobs: @@ -34,7 +36,7 @@ jobs: CIBW_SKIP: "pp* cp38-macosx_*" - name: Build sdist - if: runner.os == 'Linux' + if: runner.os == 'Linux' && runner.arch == 'X64' run: | pip install build python -m build -s -o wheelhouse @@ -44,3 +46,43 @@ jobs: with: name: wheels-${{ matrix.os }} path: wheelhouse + + # combine all wheels into one artifact + combine_wheels: + needs: [build_wheels] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + # unpacks all CIBW artifacts into dist/ + pattern: wheels-* + path: dist + merge-multiple: true + + - name: Upload Wheels to artifact + uses: actions/upload-artifact@v4 + with: + name: wheels-all + path: dist + + # https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml + upload_pypi: + needs: [build_wheels] + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + if: github.event_name == 'release' && github.event.action == 'published' + # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this) + # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + steps: + - uses: actions/download-artifact@v4 + with: + # unpacks all CIBW artifacts into dist/ + pattern: wheels-* + path: dist + merge-multiple: true + + - uses: pypa/gh-action-pypi-publish@release/v1 + #with: + # To test: repository-url: https://test.pypi.org/legacy/ From e6445d3b922ca0b9bc82695dd9d1c1529763095a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Jun 2025 09:38:38 +0900 Subject: [PATCH 1162/1172] v1.1.1rc1 --- ChangeLog.rst | 7 +++++++ docs/conf.py | 6 +++--- msgpack/__init__.py | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 863c6b2..0f67502 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,10 @@ +1.1.1rc1 +======== + +Release Date: 2025-06-06 + +* Update Cython to 3.1.1 and cibuildwheel to 2.23.3. + 1.1.0 ===== diff --git a/docs/conf.py b/docs/conf.py index ab0ad3c..28116cd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,9 +12,9 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#import os -#import sys -#sys.path.insert(0, os.path.abspath('..')) +# import os +# import sys +# sys.path.insert(0, os.path.abspath('..')) # -- General configuration ----------------------------------------------------- diff --git a/msgpack/__init__.py b/msgpack/__init__.py index b615105..b8d83da 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ import os from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 0) -__version__ = "1.1.0" +version = (1, 1, 1, "rc1") +__version__ = "1.1.1rc1" if os.environ.get("MSGPACK_PUREPYTHON"): From 42f056f3cfaf2e3ec220db2f864e7613d433ad48 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 13 Jun 2025 15:41:08 +0900 Subject: [PATCH 1163/1172] v1.1.1 --- ChangeLog.rst | 7 +++++++ msgpack/__init__.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 0f67502..418c444 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,10 @@ +1.1.1 +===== + +Release Date: 2025-06-13 + +* No change from 1.1.1rc1. + 1.1.1rc1 ======== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index b8d83da..ad68271 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ import os from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 1, "rc1") -__version__ = "1.1.1rc1" +version = (1, 1, 1) +__version__ = "1.1.1" if os.environ.get("MSGPACK_PUREPYTHON"): From d9873dab049c60d5fd553223b9be10f1dcb56929 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 26 Jul 2025 10:59:05 +0900 Subject: [PATCH 1164/1172] ci: update cibuildwheel and drop Python 3.8 (#642) --- .github/workflows/test.yml | 2 +- .github/workflows/wheel.yml | 4 ++-- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 23d221c..6089479 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 686d7dd..1260036 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -29,11 +29,11 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.23.3 + uses: pypa/cibuildwheel@v3.1.1 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" - CIBW_SKIP: "pp* cp38-macosx_*" + CIBW_SKIP: "pp* cp38-*" - name: Build sdist if: runner.os == 'Linux' && runner.arch == 'X64' diff --git a/requirements.txt b/requirements.txt index 78a2f38..5d2e20b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython~=3.1.1 +Cython~=3.1.2 From c2a9f1fda5401de2193c43d7a4f93bfc0b2fc1e6 Mon Sep 17 00:00:00 2001 From: MS-GITS <137760120+Greenie0701@users.noreply.github.com> Date: Fri, 26 Sep 2025 09:47:17 +0530 Subject: [PATCH 1165/1172] ci: add support for building windows on arm wheels (#643) --- .github/workflows/test.yml | 12 ++++++++---- .github/workflows/wheel.yml | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6089479..26b4ce6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,9 +9,13 @@ jobs: test: strategy: matrix: - os: ["ubuntu-latest", "windows-latest", "macos-latest"] + os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest"] py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9"] - + exclude: + - os: windows-11-arm + py: "3.9" + - os: windows-11-arm + py: "3.10" runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -29,8 +33,8 @@ jobs: - name: Prepare shell: bash run: | - pip install -U pip - pip install -r requirements.txt pytest + python -m pip install -U pip + python -m pip install -r requirements.txt pytest - name: Build shell: bash diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 1260036..fb369a1 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: # macos-13 is for intel - os: ["ubuntu-24.04", "ubuntu-24.04-arm", "windows-latest", "macos-13", "macos-latest"] + os: ["ubuntu-24.04", "ubuntu-24.04-arm", "windows-latest", "windows-11-arm", "macos-13", "macos-latest"] runs-on: ${{ matrix.os }} name: Build wheels on ${{ matrix.os }} @@ -33,7 +33,7 @@ jobs: env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" - CIBW_SKIP: "pp* cp38-*" + CIBW_SKIP: "pp* cp38-* cp39-win_arm64 cp310-win_arm64" - name: Build sdist if: runner.os == 'Linux' && runner.arch == 'X64' From 0f3c4be465b5f39ab2d6903797e57ef8a0fd097e Mon Sep 17 00:00:00 2001 From: TW Date: Wed, 8 Oct 2025 09:10:46 +0200 Subject: [PATCH 1166/1172] README: fix typos and grammar (#648) --- README.md | 64 +++++++++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 61a03c6..1f06324 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Build Status](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml/badge.svg)](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml) [![Documentation Status](https://readthedocs.org/projects/msgpack-python/badge/?version=latest)](https://msgpack-python.readthedocs.io/en/latest/?badge=latest) -## What's this +## What is this? [MessagePack](https://msgpack.org/) is an efficient binary serialization format. It lets you exchange data among multiple languages like JSON. @@ -25,9 +25,9 @@ But msgpack provides a pure Python implementation (`msgpack.fallback`) for PyPy. ### Windows -When you can't use a binary distribution, you need to install Visual Studio -or Windows SDK on Windows. -Without extension, using pure Python implementation on CPython runs slowly. +If you can't use a binary distribution, you need to install Visual Studio +or the Windows SDK on Windows. +Without the extension, the pure Python implementation on CPython runs slowly. ## How to use @@ -35,11 +35,11 @@ Without extension, using pure Python implementation on CPython runs slowly. ### One-shot pack & unpack Use `packb` for packing and `unpackb` for unpacking. -msgpack provides `dumps` and `loads` as an alias for compatibility with +msgpack provides `dumps` and `loads` as aliases for compatibility with `json` and `pickle`. -`pack` and `dump` packs to a file-like object. -`unpack` and `load` unpacks from a file-like object. +`pack` and `dump` pack to a file-like object. +`unpack` and `load` unpack from a file-like object. ```pycon >>> import msgpack @@ -73,7 +73,7 @@ for unpacked in unpacker: ``` -### Packing/unpacking of custom data type +### Packing/unpacking of custom data types It is also possible to pack/unpack custom data types. Here is an example for `datetime.datetime`. @@ -140,8 +140,8 @@ True ### Advanced unpacking control As an alternative to iteration, `Unpacker` objects provide `unpack`, -`skip`, `read_array_header` and `read_map_header` methods. The former two -read an entire message from the stream, respectively de-serialising and returning +`skip`, `read_array_header`, and `read_map_header` methods. The former two +read an entire message from the stream, respectively deserializing and returning the result, or ignoring it. The latter two methods return the number of elements in the upcoming container, so that each element in an array, or key-value pair in a map, can be unpacked or skipped individually. @@ -149,7 +149,7 @@ in a map, can be unpacked or skipped individually. ## Notes -### string and binary type in old msgpack spec +### String and binary types in the old MessagePack spec Early versions of msgpack didn't distinguish string and binary types. The type for representing both string and binary types was named **raw**. @@ -167,7 +167,7 @@ and `raw=True` options. ### ext type -To use the **ext** type, pass `msgpack.ExtType` object to packer. +To use the **ext** type, pass a `msgpack.ExtType` object to the packer. ```pycon >>> import msgpack @@ -181,26 +181,26 @@ You can use it with `default` and `ext_hook`. See below. ### Security -To unpacking data received from unreliable source, msgpack provides +When unpacking data received from an unreliable source, msgpack provides two security options. `max_buffer_size` (default: `100*1024*1024`) limits the internal buffer size. -It is used to limit the preallocated list size too. +It is also used to limit preallocated list sizes. `strict_map_key` (default: `True`) limits the type of map keys to bytes and str. -While msgpack spec doesn't limit the types of the map keys, -there is a risk of the hashdos. +While the MessagePack spec doesn't limit map key types, +there is a risk of a hash DoS. If you need to support other types for map keys, use `strict_map_key=False`. ### Performance tips -CPython's GC starts when growing allocated object. -This means unpacking may cause useless GC. -You can use `gc.disable()` when unpacking large message. +CPython's GC starts when the number of allocated objects grows. +This means unpacking may trigger unnecessary GC. +You can use `gc.disable()` when unpacking a large message. -List is the default sequence type of Python. -But tuple is lighter than list. +A list is the default sequence type in Python. +However, a tuple is lighter than a list. You can use `use_list=False` while unpacking when performance is important. @@ -208,7 +208,7 @@ You can use `use_list=False` while unpacking when performance is important. ### msgpack 0.5 -Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5. +The package name on PyPI was changed from `msgpack-python` to `msgpack` in 0.5. When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before `pip install -U msgpack`. @@ -218,25 +218,25 @@ When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` be * Python 2 support - * The extension module does not support Python 2 anymore. + * The extension module no longer supports Python 2. The pure Python implementation (`msgpack.fallback`) is used for Python 2. * msgpack 1.0.6 drops official support of Python 2.7, as pip and - GitHub Action (setup-python) no longer support Python 2.7. + GitHub Action "setup-python" no longer supports Python 2.7. * Packer * Packer uses `use_bin_type=True` by default. - Bytes are encoded in bin type in msgpack. - * The `encoding` option is removed. UTF-8 is used always. + Bytes are encoded in the bin type in MessagePack. + * The `encoding` option is removed. UTF-8 is always used. * Unpacker - * Unpacker uses `raw=False` by default. It assumes str types are valid UTF-8 string - and decode them to Python str (unicode) object. + * Unpacker uses `raw=False` by default. It assumes str values are valid UTF-8 strings + and decodes them to Python str (Unicode) objects. * `encoding` option is removed. You can use `raw=True` to support old format (e.g. unpack into bytes, not str). - * Default value of `max_buffer_size` is changed from 0 to 100 MiB to avoid DoS attack. + * The default value of `max_buffer_size` is changed from 0 to 100 MiB to avoid DoS attacks. You need to pass `max_buffer_size=0` if you have large but safe data. - * Default value of `strict_map_key` is changed to True to avoid hashdos. - You need to pass `strict_map_key=False` if you have data which contain map keys - which type is not bytes or str. + * The default value of `strict_map_key` is changed to True to avoid hash DoS. + You need to pass `strict_map_key=False` if you have data that contain map keys + whose type is neither bytes nor str. From 19b5d33ded09d5a563a677c1a3d75b6246181259 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 8 Oct 2025 17:56:20 +0900 Subject: [PATCH 1167/1172] release v1.1.2 (#649) --- .github/workflows/wheel.yml | 2 +- ChangeLog.rst | 13 +++++++++++++ msgpack/__init__.py | 4 ++-- pyproject.toml | 18 ++++-------------- requirements.txt | 2 +- test/test_buffer.py | 4 ++-- 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index fb369a1..fca321a 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -29,7 +29,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v3.1.1 + uses: pypa/cibuildwheel@v3.2.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" diff --git a/ChangeLog.rst b/ChangeLog.rst index 418c444..beeab15 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,16 @@ +1.1.2 +===== + +Release Date: 2025-10-08 + +This release does not change source code. It updates only building wheels: + +* Update Cython to v3.1.4 +* Update cibuildwheel to v3.2.0 +* Drop Python 3.8 +* Add Python 3.14 +* Add windows-arm + 1.1.1 ===== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index ad68271..f3266b7 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ import os from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 1) -__version__ = "1.1.1" +version = (1, 1, 2) +__version__ = "1.1.2" if os.environ.get("MSGPACK_PUREPYTHON"): diff --git a/pyproject.toml b/pyproject.toml index b162832..3976adc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,31 +1,21 @@ [build-system] -# 75.3.0 is the latest version supporting Python 3.8 -requires = ["setuptools >= 75.3.0"] +requires = ["setuptools >= 80.9.0"] build-backend = "setuptools.build_meta" [project] name = "msgpack" dynamic = ["version"] -# `license = "Apache-2.0"` is preferred. But keep old syntax for Python 3.8 compatibility. -# https://github.com/msgpack/msgpack-python/pull/637 -license = {text="Apache 2.0"} +license = "Apache-2.0" authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}] description = "MessagePack serializer" readme = "README.md" keywords = ["msgpack", "messagepack", "serializer", "serialization", "binary"] -requires-python = ">=3.8" +requires-python = ">=3.9" classifiers = [ "Development Status :: 5 - Production/Stable", "Operating System :: OS Independent", "Topic :: File Formats", "Intended Audience :: Developers", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] @@ -46,7 +36,7 @@ version = {attr = "msgpack.__version__"} [tool.ruff] line-length = 100 -target-version = "py38" +target-version = "py39" lint.select = [ "E", # pycodestyle "F", # Pyflakes diff --git a/requirements.txt b/requirements.txt index 5d2e20b..b544cca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython~=3.1.2 +Cython==3.1.4 diff --git a/test/test_buffer.py b/test/test_buffer.py index 2c5a14c..ca09722 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -17,7 +17,7 @@ def test_unpack_bytearray(): obj = unpackb(buf, use_list=1) assert [b"foo", b"bar"] == obj expected_type = bytes - assert all(type(s) == expected_type for s in obj) + assert all(type(s) is expected_type for s in obj) def test_unpack_memoryview(): @@ -26,7 +26,7 @@ def test_unpack_memoryview(): obj = unpackb(view, use_list=1) assert [b"foo", b"bar"] == obj expected_type = bytes - assert all(type(s) == expected_type for s in obj) + assert all(type(s) is expected_type for s in obj) def test_packer_getbuffer(): From ef4f83df16e552e8c8fb12ceac55f5fda9f7f340 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Oct 2025 13:00:46 +0900 Subject: [PATCH 1168/1172] relax setuptools version (#652) --- .github/workflows/test.yml | 4 +--- MANIFEST.in | 2 +- pyproject.toml | 2 +- requirements.txt | 2 ++ 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 26b4ce6..74c311c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,6 @@ jobs: - name: Prepare shell: bash run: | - python -m pip install -U pip python -m pip install -r requirements.txt pytest - name: Build @@ -55,8 +54,7 @@ jobs: - name: build packages shell: bash run: | - pip install build - python -m build + python -m build -nv - name: upload packages uses: actions/upload-artifact@v4 diff --git a/MANIFEST.in b/MANIFEST.in index 57d84a4..6317706 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include setup.py include COPYING include README.md -recursive-include msgpack *.h *.c *.pyx *.cpp +recursive-include msgpack *.h *.c *.pyx recursive-include test *.py diff --git a/pyproject.toml b/pyproject.toml index 3976adc..3633d6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools >= 80.9.0"] +requires = ["setuptools >= 77.0.3"] build-backend = "setuptools.build_meta" [project] diff --git a/requirements.txt b/requirements.txt index b544cca..f8f8370 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ Cython==3.1.4 +setuptools==77.0.3 +build From c2546eabc414b6f232d41b7d9e51b9ff9fa93296 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Oct 2025 15:28:01 +0900 Subject: [PATCH 1169/1172] update setuptools requirements to >=78.1.1 (#653) https://github.com/advisories/GHSA-5rjg-fvgr-3xxf --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3633d6b..a4c2172 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools >= 77.0.3"] +requires = ["setuptools >= 78.1.1"] build-backend = "setuptools.build_meta" [project] diff --git a/requirements.txt b/requirements.txt index f8f8370..2f1c55b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ Cython==3.1.4 -setuptools==77.0.3 +setuptools==78.1.1 build From af456409709f58add777090bdbe6a4696879e49d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Oct 2025 15:53:08 +0900 Subject: [PATCH 1170/1172] cython: freethreading_compatible (#654) ``` $ v3/bin/python -VV Python 3.14.0 free-threading build (main, Oct 7 2025, 15:35:12) [Clang 20.1.4 ] $ v3/bin/python -c 'import sys,msgpack; print(sys._is_gil_enabled())' False ``` --- .github/workflows/test.yml | 2 +- msgpack/_cmsgpack.pyx | 3 ++- msgpack/_packer.pyx | 10 ++++++++-- msgpack/_unpacker.pyx | 11 +++++++++-- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 74c311c..20410fe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest"] - py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9"] + py: ["3.14", "3.14t", "3.13", "3.12", "3.11", "3.10", "3.9"] exclude: - os: windows-11-arm py: "3.9" diff --git a/msgpack/_cmsgpack.pyx b/msgpack/_cmsgpack.pyx index 1faaac3..9680b31 100644 --- a/msgpack/_cmsgpack.pyx +++ b/msgpack/_cmsgpack.pyx @@ -1,5 +1,6 @@ -# coding: utf-8 #cython: embedsignature=True, c_string_encoding=ascii, language_level=3 +#cython: freethreading_compatible = True +import cython from cpython.datetime cimport import_datetime, datetime_new import_datetime() diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 402b694..94d1462 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -1,5 +1,3 @@ -# coding: utf-8 - from cpython cimport * from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact from cpython.datetime cimport ( @@ -129,6 +127,7 @@ cdef class Packer: if self.exports > 0: raise BufferError("Existing exports of data: Packer cannot be changed") + @cython.critical_section def __init__(self, *, default=None, bint use_single_float=False, bint autoreset=True, bint use_bin_type=True, bint strict_types=False, bint datetime=False, unicode_errors=None, @@ -269,6 +268,7 @@ cdef class Packer: return ret return self._pack_inner(o, 0, nest_limit) + @cython.critical_section def pack(self, object obj): cdef int ret self._check_exports() @@ -284,6 +284,7 @@ cdef class Packer: self.pk.length = 0 return buf + @cython.critical_section def pack_ext_type(self, typecode, data): self._check_exports() if len(data) > ITEM_LIMIT: @@ -291,6 +292,7 @@ cdef class Packer: msgpack_pack_ext(&self.pk, typecode, len(data)) msgpack_pack_raw_body(&self.pk, data, len(data)) + @cython.critical_section def pack_array_header(self, long long size): self._check_exports() if size > ITEM_LIMIT: @@ -301,6 +303,7 @@ cdef class Packer: self.pk.length = 0 return buf + @cython.critical_section def pack_map_header(self, long long size): self._check_exports() if size > ITEM_LIMIT: @@ -311,6 +314,7 @@ cdef class Packer: self.pk.length = 0 return buf + @cython.critical_section def pack_map_pairs(self, object pairs): """ Pack *pairs* as msgpack map type. @@ -331,6 +335,7 @@ cdef class Packer: self.pk.length = 0 return buf + @cython.critical_section def reset(self): """Reset internal buffer. @@ -339,6 +344,7 @@ cdef class Packer: self._check_exports() self.pk.length = 0 + @cython.critical_section def bytes(self): """Return internal buffer contents as bytes object""" return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 34ff330..f0cf96d 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -1,5 +1,3 @@ -# coding: utf-8 - from cpython cimport * cdef extern from "Python.h": ctypedef struct PyObject @@ -324,6 +322,7 @@ cdef class Unpacker: PyMem_Free(self.buf) self.buf = NULL + @cython.critical_section def __init__(self, file_like=None, *, Py_ssize_t read_size=0, bint use_list=True, bint raw=False, int timestamp=0, bint strict_map_key=True, object object_hook=None, object object_pairs_hook=None, object list_hook=None, @@ -384,6 +383,7 @@ cdef class Unpacker: max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) + @cython.critical_section def feed(self, object next_bytes): """Append `next_bytes` to internal buffer.""" cdef Py_buffer pybuff @@ -484,6 +484,7 @@ cdef class Unpacker: else: raise ValueError("Unpack failed: error = %d" % (ret,)) + @cython.critical_section def read_bytes(self, Py_ssize_t nbytes): """Read a specified number of raw bytes from the stream""" cdef Py_ssize_t nread @@ -496,6 +497,7 @@ cdef class Unpacker: self.stream_offset += nread return ret + @cython.critical_section def unpack(self): """Unpack one object @@ -503,6 +505,7 @@ cdef class Unpacker: """ return self._unpack(unpack_construct) + @cython.critical_section def skip(self): """Read and ignore one object, returning None @@ -510,6 +513,7 @@ cdef class Unpacker: """ return self._unpack(unpack_skip) + @cython.critical_section def read_array_header(self): """assuming the next object is an array, return its size n, such that the next n unpack() calls will iterate over its contents. @@ -518,6 +522,7 @@ cdef class Unpacker: """ return self._unpack(read_array_header) + @cython.critical_section def read_map_header(self): """assuming the next object is a map, return its size n, such that the next n * 2 unpack() calls will iterate over its key-value pairs. @@ -526,6 +531,7 @@ cdef class Unpacker: """ return self._unpack(read_map_header) + @cython.critical_section def tell(self): """Returns the current position of the Unpacker in bytes, i.e., the number of bytes that were read from the input, also the starting @@ -536,6 +542,7 @@ cdef class Unpacker: def __iter__(self): return self + @cython.critical_section def __next__(self): return self._unpack(unpack_construct, 1) From c1ecd23dbfaa6c6f2a408e7e5c4727c6bdce7608 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Oct 2025 18:33:23 +0900 Subject: [PATCH 1171/1172] drop Python 3.9 (#656) --- .github/workflows/test.yml | 8 +++----- .github/workflows/wheel.yml | 6 +++--- pyproject.toml | 4 ++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 20410fe..6b1664a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,10 +10,8 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest"] - py: ["3.14", "3.14t", "3.13", "3.12", "3.11", "3.10", "3.9"] + py: ["3.14", "3.14t", "3.13", "3.12", "3.11", "3.10"] exclude: - - os: windows-11-arm - py: "3.9" - os: windows-11-arm py: "3.10" runs-on: ${{ matrix.os }} @@ -21,10 +19,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.py }} allow-prereleases: true diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index fca321a..be54e5e 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -17,8 +17,8 @@ jobs: name: Build wheels on ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 with: python-version: "3.x" cache: "pip" @@ -33,7 +33,7 @@ jobs: env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" - CIBW_SKIP: "pp* cp38-* cp39-win_arm64 cp310-win_arm64" + CIBW_SKIP: "pp* cp38-* cp39-* cp310-win_arm64" - name: Build sdist if: runner.os == 'Linux' && runner.arch == 'X64' diff --git a/pyproject.toml b/pyproject.toml index a4c2172..c69d5a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}] description = "MessagePack serializer" readme = "README.md" keywords = ["msgpack", "messagepack", "serializer", "serialization", "binary"] -requires-python = ">=3.9" +requires-python = ">=3.10" classifiers = [ "Development Status :: 5 - Production/Stable", "Operating System :: OS Independent", @@ -36,7 +36,7 @@ version = {attr = "msgpack.__version__"} [tool.ruff] line-length = 100 -target-version = "py39" +target-version = "py310" lint.select = [ "E", # pycodestyle "F", # Pyflakes From f9806368ae302722f095ea884436b20ed1cddf33 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 1 Dec 2025 14:16:03 +0900 Subject: [PATCH 1172/1172] update cython and cibuildwheel (#658) --- .github/workflows/wheel.yml | 4 ++-- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index be54e5e..531abbc 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -17,7 +17,7 @@ jobs: name: Build wheels on ${{ matrix.os }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: python-version: "3.x" @@ -29,7 +29,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v3.2.0 + uses: pypa/cibuildwheel@v3.3.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" diff --git a/requirements.txt b/requirements.txt index 2f1c55b..9e4643b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -Cython==3.1.4 +Cython==3.2.1 setuptools==78.1.1 build